From a2ba78411cafaac7df8111f4684666de6e30b63d Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Fri, 16 Mar 2018 11:47:39 +0000 Subject: [PATCH 1/7] [REF] module_auto_update: Step 1, move all deprecated stuff to deprecated files - Files are clearly suffixed with `_deprecated` so we know those features have no support nor migrations. - Views are removed, since updating from UI was too buggy to support it anymore. --- module_auto_update/__manifest__.py | 3 +- ...cron_data.xml => cron_data_deprecated.xml} | 0 module_auto_update/models/__init__.py | 2 +- .../{module.py => module_deprecated.py} | 0 module_auto_update/tests/__init__.py | 4 +- ...st_module.py => test_module_deprecated.py} | 30 ++++++------ ...e.py => test_module_upgrade_deprecated.py} | 0 module_auto_update/views/module_views.xml | 49 ------------------- module_auto_update/wizards/__init__.py | 2 +- ...pgrade.py => module_upgrade_deprecated.py} | 0 10 files changed, 21 insertions(+), 69 deletions(-) rename module_auto_update/data/{cron_data.xml => cron_data_deprecated.xml} (100%) rename module_auto_update/models/{module.py => module_deprecated.py} (100%) rename module_auto_update/tests/{test_module.py => test_module_deprecated.py} (93%) rename module_auto_update/tests/{test_module_upgrade.py => test_module_upgrade_deprecated.py} (100%) delete mode 100644 module_auto_update/views/module_views.xml rename module_auto_update/wizards/{module_upgrade.py => module_upgrade_deprecated.py} (100%) diff --git a/module_auto_update/__manifest__.py b/module_auto_update/__manifest__.py index 50a6578ae..93cdcaecf 100644 --- a/module_auto_update/__manifest__.py +++ b/module_auto_update/__manifest__.py @@ -24,7 +24,6 @@ 'base', ], 'data': [ - 'views/module_views.xml', - 'data/cron_data.xml', + 'data/cron_data_deprecated.xml', ], } diff --git a/module_auto_update/data/cron_data.xml b/module_auto_update/data/cron_data_deprecated.xml similarity index 100% rename from module_auto_update/data/cron_data.xml rename to module_auto_update/data/cron_data_deprecated.xml diff --git a/module_auto_update/models/__init__.py b/module_auto_update/models/__init__.py index e5ee3ea66..19086b02d 100644 --- a/module_auto_update/models/__init__.py +++ b/module_auto_update/models/__init__.py @@ -1,3 +1,3 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). -from . import module +from . import module_deprecated diff --git a/module_auto_update/models/module.py b/module_auto_update/models/module_deprecated.py similarity index 100% rename from module_auto_update/models/module.py rename to module_auto_update/models/module_deprecated.py diff --git a/module_auto_update/tests/__init__.py b/module_auto_update/tests/__init__.py index 06952e34e..6c2171bab 100644 --- a/module_auto_update/tests/__init__.py +++ b/module_auto_update/tests/__init__.py @@ -1,4 +1,4 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). -from . import test_module -from . import test_module_upgrade +from . import test_module_deprecated +from . import test_module_upgrade_deprecated diff --git a/module_auto_update/tests/test_module.py b/module_auto_update/tests/test_module_deprecated.py similarity index 93% rename from module_auto_update/tests/test_module.py rename to module_auto_update/tests/test_module_deprecated.py index 08fda9841..51f347677 100644 --- a/module_auto_update/tests/test_module.py +++ b/module_auto_update/tests/test_module_deprecated.py @@ -19,7 +19,7 @@ try: except ImportError: _logger.debug('Cannot `import checksumdir`.') -model = 'odoo.addons.module_auto_update.models.module' +model = 'odoo.addons.module_auto_update.models.module_deprecated' class EndTestException(Exception): @@ -179,19 +179,21 @@ class TestModule(TransactionCase): def test_create(self): """It should call _store_checksum_installed method""" _store_checksum_installed_mock = mock.MagicMock() - self.env['ir.module.module']._patch_method( - '_store_checksum_installed', - _store_checksum_installed_mock, - ) - vals = { - 'name': 'module_auto_update_test_module', - 'state': 'installed', - } - self.create_test_module(vals) - _store_checksum_installed_mock.assert_called_once_with(vals) - self.env['ir.module.module']._revert_method( - '_store_checksum_installed', - ) + try: + self.env['ir.module.module']._patch_method( + '_store_checksum_installed', + _store_checksum_installed_mock, + ) + vals = { + 'name': 'module_auto_update_test_module', + 'state': 'installed', + } + self.create_test_module(vals) + _store_checksum_installed_mock.assert_called_once_with(vals) + finally: + self.env['ir.module.module']._revert_method( + '_store_checksum_installed', + ) @mute_logger("openerp.modules.module") @mock.patch('%s.get_module_path' % model) diff --git a/module_auto_update/tests/test_module_upgrade.py b/module_auto_update/tests/test_module_upgrade_deprecated.py similarity index 100% rename from module_auto_update/tests/test_module_upgrade.py rename to module_auto_update/tests/test_module_upgrade_deprecated.py diff --git a/module_auto_update/views/module_views.xml b/module_auto_update/views/module_views.xml deleted file mode 100644 index 78a0be51e..000000000 --- a/module_auto_update/views/module_views.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - updates.module.search - ir.module.module - - - - - - - - - - - Open Updates and Update Apps List Server Action - - - if model.update_list(): - action = { - 'name': 'Updates', - 'type': 'ir.actions.act_window', - 'res_model': 'ir.module.module', - 'view_type': 'form', - 'view_mode': 'tree,form', - 'target': 'main', - 'context': '{"search_default_scheduled_upgrades": 1}', - } - - - - - - - - - - - - - - diff --git a/module_auto_update/wizards/__init__.py b/module_auto_update/wizards/__init__.py index 0448de3cf..bcaca7966 100644 --- a/module_auto_update/wizards/__init__.py +++ b/module_auto_update/wizards/__init__.py @@ -1,3 +1,3 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). -from . import module_upgrade +from . import module_upgrade_deprecated diff --git a/module_auto_update/wizards/module_upgrade.py b/module_auto_update/wizards/module_upgrade_deprecated.py similarity index 100% rename from module_auto_update/wizards/module_upgrade.py rename to module_auto_update/wizards/module_upgrade_deprecated.py From b08ea7e0a231742b1fab151175c647f206d745c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Bidoul=20=28ACSONE=29?= Date: Tue, 27 Feb 2018 23:45:55 +0100 Subject: [PATCH 2/7] [REF] module_auto_update: Step 2, add new API This code comes from the module_checksum_upgrade proposal at https://github.com/OCA/server-tools/pull/1176. * [ADD] module_checksum_upgrade It provides the core mechanism of module_auto_update without the cron nor any change to the standard upgrade mechanism. Instead it provides an API on which module_auto_update can build, as well as a method which can be called from a script to run the upgrade of modules for which the checksum has changed. * [IMP] refactor module_auto_update Make it depend on module_checksum_upgrade which provides the core mechanisms of managing the checksums. module_auto_update makes it automatic. * [IMP] module_checksum_upgrade: better exclusion mechanism Ignore files based on exclude patterns. Ignore uninstalled languages. Better default for patterns to ignore (*.pyc,*.pyo,*.pot,static/*) For better control on the hashing mechanism implement our own: it's quite easy, and the checksumdir module used previously had no test. * [MIG] module_auto_update: adapt to new checksum mechanism * [IMP] module_checksum_upgrade: raise in case of incomplete upgrade * [IMP] module_checksum_upgrade: improve default exclusion pattern * [IMP] module_checksum_upgrade: control translations overwrite * [IMP] module_checksum_upgrade: one more test * [IMP] module_checksum_upgrade: credits [ci skip] --- module_auto_update/README.rst | 43 ++-- module_auto_update/__manifest__.py | 6 +- module_auto_update/addon_hash.py | 45 ++++ module_auto_update/hooks.py | 6 +- module_auto_update/models/__init__.py | 1 + module_auto_update/models/module.py | 148 +++++++++++++ .../models/module_deprecated.py | 44 ++-- module_auto_update/tests/__init__.py | 2 + .../tests/sample_module/README.rst | 1 + .../tests/sample_module/data/f1.xml | 1 + .../tests/sample_module/data/f2.xml | 1 + .../tests/sample_module/i18n/en.po | 1 + .../tests/sample_module/i18n/en_US.po | 1 + .../tests/sample_module/i18n/fr.po | 1 + .../tests/sample_module/i18n/fr_BE.po | 1 + .../tests/sample_module/i18n/test.pot | 1 + .../tests/sample_module/i18n_extra/en.po | 1 + .../tests/sample_module/i18n_extra/fr.po | 1 + .../tests/sample_module/i18n_extra/nl_NL.po | 1 + .../tests/sample_module/models/stuff.py | 1 + .../tests/sample_module/models/stuff.pyc | Bin 0 -> 109 bytes .../tests/sample_module/static/src/some.js | 1 + module_auto_update/tests/test_addon_hash.py | 67 ++++++ module_auto_update/tests/test_module.py | 203 ++++++++++++++++++ .../tests/test_module_deprecated.py | 15 +- 25 files changed, 533 insertions(+), 60 deletions(-) create mode 100644 module_auto_update/addon_hash.py create mode 100644 module_auto_update/models/module.py create mode 100644 module_auto_update/tests/sample_module/README.rst create mode 100644 module_auto_update/tests/sample_module/data/f1.xml create mode 100644 module_auto_update/tests/sample_module/data/f2.xml create mode 100644 module_auto_update/tests/sample_module/i18n/en.po create mode 100644 module_auto_update/tests/sample_module/i18n/en_US.po create mode 100644 module_auto_update/tests/sample_module/i18n/fr.po create mode 100644 module_auto_update/tests/sample_module/i18n/fr_BE.po create mode 100644 module_auto_update/tests/sample_module/i18n/test.pot create mode 100644 module_auto_update/tests/sample_module/i18n_extra/en.po create mode 100644 module_auto_update/tests/sample_module/i18n_extra/fr.po create mode 100644 module_auto_update/tests/sample_module/i18n_extra/nl_NL.po create mode 100644 module_auto_update/tests/sample_module/models/stuff.py create mode 100644 module_auto_update/tests/sample_module/models/stuff.pyc create mode 100644 module_auto_update/tests/sample_module/static/src/some.js create mode 100644 module_auto_update/tests/test_addon_hash.py create mode 100644 module_auto_update/tests/test_module.py diff --git a/module_auto_update/README.rst b/module_auto_update/README.rst index 6349fdd7b..8989df79a 100644 --- a/module_auto_update/README.rst +++ b/module_auto_update/README.rst @@ -6,31 +6,43 @@ Module Auto Update ================== -This module will automatically check for and apply module upgrades on a schedule. - -Upgrade checking is accomplished by comparing the SHA1 checksums of currently-installed modules to the checksums of corresponding modules in the addons directories. - -Installation -============ - -Prior to installing this module, you need to: - -#. Install checksumdir with ``pip install checksumdir`` -#. Ensure all installed modules are up-to-date. When installed, this module will assume the versions found in the addons directories are currently installed. +This addon provides mechanisms to compute sha1 hashes of installed addons, +and save them in the database. It also provides a method that exploits these +mechanisms to update a database by upgrading only the modules for which the +hash has changed since the last successful upgrade. Configuration ============= -The default time for checking and applying upgrades is 3:00 AM (UTC). To change this schedule, modify the "Perform Module Upgrades" scheduled action. +This module supports the following system parameters: -This module will ignore ``.pyc`` and ``.pyo`` file extensions by default. To modify this, create a ``module_auto_update.checksum_excluded_extensions`` system parameter with the desired extensions listed as comma-separated values. +* ``module_auto_update.exclude_patterns``: comma-separated list of file + name patterns to ignore when computing addon checksums. Defaults to + ``*.pyc,*.pyo,i18n/*.pot,i18n_extra/*.pot,static/*``. + Filename patterns must be compatible with the python ``fnmatch`` function. + +In addition to the above pattern, .po files corresponding to languages that +are not installed in the Odoo database are ignored when computing checksums. Usage ===== -Modules scheduled for upgrade can be viewed by clicking the "Updates" menu item in the Apps sidebar. +The main method provided by this module is ``upgrade_changed_checksum`` +on ``ir.module.module``. It runs a database upgrade for all installed +modules for which the hash has changed since the last successful +run of this method. On success it saves the hashes in the database. + +The first time this method is invoked after installing the module, it +runs an upgrade of all modules, because it has not saved the hashes yet. +This is by design, priviledging safety. Should this be an issue, +the method ``_save_installed_checksums`` can be invoked in a situation +where one is sure all modules on disk are installed and up-to-date in the +database. + +An easy way to invoke this upgrade mechanism is by issuing the following +in an Odoo shell session:: -To perform upgrades manually, click the "Apply Scheduled Upgrades" menu item in the Apps sidebar. + env['ir.module.module'].upgrade_changed_checksum() .. image:: https://odoo-community.org/website/image/ir.attachment/5784_f2813bd/datas :alt: Try me on Runbot @@ -58,6 +70,7 @@ Contributors * Brent Hughes * Juan José Scarafía * Jairo Llopis +* Stéphane Bidoul (https://acsone.eu) Do not contact contributors directly about support or help with technical issues. diff --git a/module_auto_update/__manifest__.py b/module_auto_update/__manifest__.py index 93cdcaecf..4f4375fc7 100644 --- a/module_auto_update/__manifest__.py +++ b/module_auto_update/__manifest__.py @@ -10,16 +10,12 @@ 'author': 'LasLabs, ' 'Juan José Scarafía, ' 'Tecnativa, ' + 'ACSONE SA/NV, ' 'Odoo Community Association (OCA)', 'license': 'LGPL-3', 'application': False, 'installable': True, 'post_init_hook': 'post_init_hook', - 'external_dependencies': { - 'python': [ - 'checksumdir', - ], - }, 'depends': [ 'base', ], diff --git a/module_auto_update/addon_hash.py b/module_auto_update/addon_hash.py new file mode 100644 index 000000000..dea52b4f2 --- /dev/null +++ b/module_auto_update/addon_hash.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 ACSONE SA/NV. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +from fnmatch import fnmatch +import hashlib +import os + + +def _fnmatch(filename, patterns): + for pattern in patterns: + if fnmatch(filename, pattern): + return True + return False + + +def _walk(top, exclude_patterns, keep_langs): + keep_langs = {l.split('_')[0] for l in keep_langs} + for dirpath, dirnames, filenames in os.walk(top): + dirnames.sort() + reldir = os.path.relpath(dirpath, top) + if reldir == '.': + reldir = '' + for filename in sorted(filenames): + filepath = os.path.join(reldir, filename) + if _fnmatch(filepath, exclude_patterns): + continue + if keep_langs and reldir in {'i18n', 'i18n_extra'}: + basename, ext = os.path.splitext(filename) + if ext == '.po': + if basename.split('_')[0] not in keep_langs: + continue + yield filepath + + +def addon_hash(top, exclude_patterns, keep_langs): + """Compute a sha1 digest of file contents.""" + m = hashlib.sha1() + for filepath in _walk(top, exclude_patterns, keep_langs): + # hash filename so empty files influence the hash + m.update(filepath.encode('utf-8')) + # hash file content + with open(os.path.join(top, filepath), 'rb') as f: + m.update(f.read()) + return m.hexdigest() diff --git a/module_auto_update/hooks.py b/module_auto_update/hooks.py index 56d60c6ef..396f78707 100644 --- a/module_auto_update/hooks.py +++ b/module_auto_update/hooks.py @@ -6,8 +6,4 @@ from odoo import SUPERUSER_ID, api def post_init_hook(cr, registry): env = api.Environment(cr, SUPERUSER_ID, {}) - installed_modules = env['ir.module.module'].search([ - ('state', '=', 'installed'), - ]) - for r in installed_modules: - r.checksum_installed = r.checksum_dir + env['ir.module.module']._save_installed_checksums() diff --git a/module_auto_update/models/__init__.py b/module_auto_update/models/__init__.py index 19086b02d..53c05a539 100644 --- a/module_auto_update/models/__init__.py +++ b/module_auto_update/models/__init__.py @@ -1,3 +1,4 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from . import module from . import module_deprecated diff --git a/module_auto_update/models/module.py b/module_auto_update/models/module.py new file mode 100644 index 000000000..334ec9d16 --- /dev/null +++ b/module_auto_update/models/module.py @@ -0,0 +1,148 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 LasLabs Inc. +# Copyright 2018 ACSONE SA/NV. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import json +import logging +import os + +from openerp import api, exceptions, models, tools +from openerp.modules.module import get_module_path + +from ..addon_hash import addon_hash + +PARAM_INSTALLED_CHECKSUMS = \ + 'module_auto_update.installed_checksums' +PARAM_EXCLUDE_PATTERNS = \ + 'module_auto_update.exclude_patterns' +DEFAULT_EXCLUDE_PATTERNS = \ + '*.pyc,*.pyo,i18n/*.pot,i18n_extra/*.pot,static/*' + +_logger = logging.getLogger(__name__) + + +class IncompleteUpgradeError(exceptions.UserError): + pass + + +class Module(models.Model): + _inherit = 'ir.module.module' + + @api.multi + def _get_checksum_dir(self): + self.ensure_one() + + exclude_patterns = self.env["ir.config_parameter"].get_param( + PARAM_EXCLUDE_PATTERNS, + DEFAULT_EXCLUDE_PATTERNS, + ) + exclude_patterns = [p.strip() for p in exclude_patterns.split(',')] + keep_langs = self.env['res.lang'].search([]).mapped('code') + + module_path = get_module_path(self.name) + if module_path and os.path.isdir(module_path): + checksum_dir = addon_hash( + module_path, + exclude_patterns, + keep_langs, + ) + else: + checksum_dir = False + + return checksum_dir + + @api.model + def _get_saved_checksums(self): + Icp = self.env['ir.config_parameter'] + return json.loads(Icp.get_param(PARAM_INSTALLED_CHECKSUMS, '{}')) + + @api.model + def _save_checksums(self, checksums): + Icp = self.env['ir.config_parameter'] + Icp.set_param(PARAM_INSTALLED_CHECKSUMS, json.dumps(checksums)) + + @api.model + def _save_installed_checksums(self): + checksums = {} + installed_modules = self.search([('state', '=', 'installed')]) + for module in installed_modules: + checksums[module.name] = module._get_checksum_dir() + self._save_checksums(checksums) + + @api.model + def _get_modules_partially_installed(self): + return self.search([ + ('state', 'in', ('to install', 'to remove', 'to upgrade')), + ]) + + @api.model + def _get_modules_with_changed_checksum(self): + saved_checksums = self._get_saved_checksums() + installed_modules = self.search([('state', '=', 'installed')]) + return installed_modules.filtered( + lambda r: r._get_checksum_dir() != saved_checksums.get(r.name), + ) + + @api.model + def upgrade_changed_checksum(self, overwrite_existing_translations=False): + """Run an upgrade of the database, upgrading only changed modules. + + Installed modules for which the checksum has changed since the + last successful run of this method are marked "to upgrade", + then the normal Odoo scheduled upgrade process + is launched. + + If there is no module with a changed checksum, and no module in state + other than installed, uninstalled, uninstallable, this method does + nothing, otherwise the normal Odoo upgrade process is launched. + + After a successful upgrade, the checksums of installed modules are + saved. + + In case of error during the upgrade, an exception is raised. + If any module remains to upgrade or to uninstall after the upgrade + process, an exception is raised as well. + + Note: this method commits the current transaction at each important + step, it is therefore not intended to be run as part of a + larger transaction. + """ + _logger.info( + "Checksum upgrade starting (i18n-overwrite=%s)...", + overwrite_existing_translations + ) + + tools.config['overwrite_existing_translations'] = \ + overwrite_existing_translations + + _logger.info("Updating modules list...") + self.update_list() + changed_modules = self._get_modules_with_changed_checksum() + if not changed_modules and not self._get_modules_partially_installed(): + _logger.info("No checksum change detected in installed modules " + "and all modules installed, nothing to do.") + return + _logger.info("Marking the following modules to upgrade, " + "for their checksums changed: %s...", + ','.join(changed_modules.mapped('name'))) + changed_modules.button_upgrade() + self.env.cr.commit() # pylint: disable=invalid-commit + + _logger.info("Upgrading...") + self.env['base.module.upgrade'].upgrade_module() + self.env.cr.commit() # pylint: disable=invalid-commit + + _logger.info("Upgrade successful, updating checksums...") + self._save_installed_checksums() + self.env.cr.commit() # pylint: disable=invalid-commit + + partial_modules = self._get_modules_partially_installed() + if partial_modules: + raise IncompleteUpgradeError( + "Checksum upgrade successful " + "but incomplete for the following modules: %s" % + ','.join(partial_modules.mapped('name')) + ) + + _logger.info("Checksum upgrade complete.") diff --git a/module_auto_update/models/module_deprecated.py b/module_auto_update/models/module_deprecated.py index 271bc5700..5f1f5b8e2 100644 --- a/module_auto_update/models/module_deprecated.py +++ b/module_auto_update/models/module_deprecated.py @@ -1,16 +1,7 @@ # Copyright 2017 LasLabs Inc. # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). -import logging - from odoo import api, fields, models -from odoo.modules.module import get_module_path - -_logger = logging.getLogger(__name__) -try: - from checksumdir import dirhash -except ImportError: - _logger.debug('Cannot `import checksumdir`.') class Module(models.Model): @@ -19,26 +10,27 @@ class Module(models.Model): checksum_dir = fields.Char( compute='_compute_checksum_dir', ) - checksum_installed = fields.Char() + checksum_installed = fields.Char( + compute='_compute_checksum_installed', + inverse='_inverse_checksum_installed', + store=False, + ) @api.depends('name') def _compute_checksum_dir(self): - exclude = self.env["ir.config_parameter"].get_param( - "module_auto_update.checksum_excluded_extensions", - "pyc,pyo", - ).split(",") - - for r in self: - try: - r.checksum_dir = dirhash( - get_module_path(r.name), - 'sha1', - excluded_extensions=exclude, - ) - except TypeError: - _logger.debug( - "Cannot compute dir hash for %s, module not found", - r.display_name) + for rec in self: + rec.checksum_dir = rec._get_checksum_dir() + + def _compute_checksum_installed(self): + saved_checksums = self._get_saved_checksums() + for rec in self: + rec.checksum_installed = saved_checksums.get(rec.name, False) + + def _inverse_checksum_installed(self): + saved_checksums = self._get_saved_checksums() + for rec in self: + saved_checksums[rec.name] = rec.checksum_installed + self._save_installed_checksums() @api.multi def _store_checksum_installed(self, vals): diff --git a/module_auto_update/tests/__init__.py b/module_auto_update/tests/__init__.py index 6c2171bab..54f7cc144 100644 --- a/module_auto_update/tests/__init__.py +++ b/module_auto_update/tests/__init__.py @@ -1,4 +1,6 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +from . import test_addon_hash +from . import test_module from . import test_module_deprecated from . import test_module_upgrade_deprecated diff --git a/module_auto_update/tests/sample_module/README.rst b/module_auto_update/tests/sample_module/README.rst new file mode 100644 index 000000000..048382e52 --- /dev/null +++ b/module_auto_update/tests/sample_module/README.rst @@ -0,0 +1 @@ +Test data for addon_hash module. diff --git a/module_auto_update/tests/sample_module/data/f1.xml b/module_auto_update/tests/sample_module/data/f1.xml new file mode 100644 index 000000000..77a8d9d78 --- /dev/null +++ b/module_auto_update/tests/sample_module/data/f1.xml @@ -0,0 +1 @@ + diff --git a/module_auto_update/tests/sample_module/data/f2.xml b/module_auto_update/tests/sample_module/data/f2.xml new file mode 100644 index 000000000..77a8d9d78 --- /dev/null +++ b/module_auto_update/tests/sample_module/data/f2.xml @@ -0,0 +1 @@ + diff --git a/module_auto_update/tests/sample_module/i18n/en.po b/module_auto_update/tests/sample_module/i18n/en.po new file mode 100644 index 000000000..c8afcebdf --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n/en.po @@ -0,0 +1 @@ +en text diff --git a/module_auto_update/tests/sample_module/i18n/en_US.po b/module_auto_update/tests/sample_module/i18n/en_US.po new file mode 100644 index 000000000..7741b83a3 --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n/en_US.po @@ -0,0 +1 @@ +en_US diff --git a/module_auto_update/tests/sample_module/i18n/fr.po b/module_auto_update/tests/sample_module/i18n/fr.po new file mode 100644 index 000000000..527e861b3 --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n/fr.po @@ -0,0 +1 @@ +fr diff --git a/module_auto_update/tests/sample_module/i18n/fr_BE.po b/module_auto_update/tests/sample_module/i18n/fr_BE.po new file mode 100644 index 000000000..961231717 --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n/fr_BE.po @@ -0,0 +1 @@ +fr_BE diff --git a/module_auto_update/tests/sample_module/i18n/test.pot b/module_auto_update/tests/sample_module/i18n/test.pot new file mode 100644 index 000000000..eb1ae458f --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n/test.pot @@ -0,0 +1 @@ +... diff --git a/module_auto_update/tests/sample_module/i18n_extra/en.po b/module_auto_update/tests/sample_module/i18n_extra/en.po new file mode 100644 index 000000000..c574d073d --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n_extra/en.po @@ -0,0 +1 @@ +en diff --git a/module_auto_update/tests/sample_module/i18n_extra/fr.po b/module_auto_update/tests/sample_module/i18n_extra/fr.po new file mode 100644 index 000000000..527e861b3 --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n_extra/fr.po @@ -0,0 +1 @@ +fr diff --git a/module_auto_update/tests/sample_module/i18n_extra/nl_NL.po b/module_auto_update/tests/sample_module/i18n_extra/nl_NL.po new file mode 100644 index 000000000..85b15b659 --- /dev/null +++ b/module_auto_update/tests/sample_module/i18n_extra/nl_NL.po @@ -0,0 +1 @@ +nl_NL diff --git a/module_auto_update/tests/sample_module/models/stuff.py b/module_auto_update/tests/sample_module/models/stuff.py new file mode 100644 index 000000000..c040fa67d --- /dev/null +++ b/module_auto_update/tests/sample_module/models/stuff.py @@ -0,0 +1 @@ +1+1 diff --git a/module_auto_update/tests/sample_module/models/stuff.pyc b/module_auto_update/tests/sample_module/models/stuff.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2050f52c7c2e2c08bfb655555c80e4342a48a0c2 GIT binary patch literal 109 zcmZSn%**Avesxqb0~9a;X$K%K<^U2YObm=Ej10jV%s@^iBaraR1S!w}Vsrwmp}3?p ZElsbWvIL~tCO1E&G$+*#q^}sH0{{jG4&DF& literal 0 HcmV?d00001 diff --git a/module_auto_update/tests/sample_module/static/src/some.js b/module_auto_update/tests/sample_module/static/src/some.js new file mode 100644 index 000000000..64797d825 --- /dev/null +++ b/module_auto_update/tests/sample_module/static/src/some.js @@ -0,0 +1 @@ +/* javascript */ diff --git a/module_auto_update/tests/test_addon_hash.py b/module_auto_update/tests/test_addon_hash.py new file mode 100644 index 000000000..7523b8ec4 --- /dev/null +++ b/module_auto_update/tests/test_addon_hash.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 ACSONE SA/NV. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import os +import unittest + +from .. import addon_hash +from ..models.module import DEFAULT_EXCLUDE_PATTERNS + + +class TestAddonHash(unittest.TestCase): + + def setUp(self): + super(TestAddonHash, self).setUp() + self.sample_dir = os.path.join( + os.path.dirname(__file__), + 'sample_module', + ) + + def test_basic(self): + files = list(addon_hash._walk( + self.sample_dir, + exclude_patterns=[], + keep_langs=[], + )) + self.assertEqual(files, [ + 'README.rst', + 'data/f1.xml', + 'data/f2.xml', + 'i18n/en.po', + 'i18n/en_US.po', + 'i18n/fr.po', + 'i18n/fr_BE.po', + 'i18n/test.pot', + 'i18n_extra/en.po', + 'i18n_extra/fr.po', + 'i18n_extra/nl_NL.po', + 'models/stuff.py', + 'models/stuff.pyc', + 'static/src/some.js', + ]) + + def test_exclude(self): + files = list(addon_hash._walk( + self.sample_dir, + exclude_patterns=DEFAULT_EXCLUDE_PATTERNS.split(','), + keep_langs=['fr_FR', 'nl'], + )) + self.assertEqual(files, [ + 'README.rst', + 'data/f1.xml', + 'data/f2.xml', + 'i18n/fr.po', + 'i18n/fr_BE.po', + 'i18n_extra/fr.po', + 'i18n_extra/nl_NL.po', + 'models/stuff.py', + ]) + + def test2(self): + checksum = addon_hash.addon_hash( + self.sample_dir, + exclude_patterns=['*.pyc', '*.pyo', '*.pot', 'static/*'], + keep_langs=['fr_FR', 'nl'], + ) + self.assertEqual(checksum, 'fecb89486c8a29d1f760cbd01c1950f6e8421b14') diff --git a/module_auto_update/tests/test_module.py b/module_auto_update/tests/test_module.py new file mode 100644 index 000000000..b4f0746c3 --- /dev/null +++ b/module_auto_update/tests/test_module.py @@ -0,0 +1,203 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 LasLabs Inc. +# Copyright 2018 ACSONE SA/NV. +# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). + +import os +import tempfile + +import mock + +from openerp.modules import get_module_path +from openerp.tests import common +from openerp.tests.common import TransactionCase + +from ..addon_hash import addon_hash +from ..models.module import IncompleteUpgradeError, DEFAULT_EXCLUDE_PATTERNS + +MODULE_NAME = 'module_auto_update' + + +class TestModule(TransactionCase): + + def setUp(self): + super(TestModule, self).setUp() + self.own_module = self.env['ir.module.module'].search([ + ('name', '=', MODULE_NAME), + ]) + self.own_dir_path = get_module_path(MODULE_NAME) + keep_langs = self.env['res.lang'].search([]).mapped('code') + self.own_checksum = addon_hash( + self.own_dir_path, + exclude_patterns=DEFAULT_EXCLUDE_PATTERNS.split(','), + keep_langs=keep_langs, + ) + self.own_writeable = os.access(self.own_dir_path, os.W_OK) + + def test_compute_checksum_dir(self): + """It should compute the directory's SHA-1 hash""" + self.assertEqual( + self.own_module._get_checksum_dir(), self.own_checksum, + 'Module directory checksum not computed properly', + ) + + def test_compute_checksum_dir_ignore_excluded(self): + """It should exclude .pyc/.pyo extensions from checksum + calculations""" + if not self.own_writeable: + self.skipTest("Own directory not writeable") + with tempfile.NamedTemporaryFile(suffix='.pyc', dir=self.own_dir_path): + self.assertEqual( + self.own_module._get_checksum_dir(), self.own_checksum, + 'SHA1 checksum does not ignore excluded extensions', + ) + + def test_compute_checksum_dir_recomputes_when_file_added(self): + """It should return a different value when a non-.pyc/.pyo file is + added to the module directory""" + if not self.own_writeable: + self.skipTest("Own directory not writeable") + with tempfile.NamedTemporaryFile(suffix='.py', dir=self.own_dir_path): + self.assertNotEqual( + self.own_module._get_checksum_dir(), self.own_checksum, + 'SHA1 checksum not recomputed', + ) + + def test_saved_checksums(self): + Imm = self.env['ir.module.module'] + base_module = Imm.search([('name', '=', 'base')]) + self.assertEqual(base_module.state, 'installed') + self.assertFalse(Imm._get_saved_checksums()) + Imm._save_installed_checksums() + saved_checksums = Imm._get_saved_checksums() + self.assertTrue(saved_checksums) + self.assertTrue(saved_checksums['base']) + + def test_get_modules_with_changed_checksum(self): + Imm = self.env['ir.module.module'] + self.assertTrue(Imm._get_modules_with_changed_checksum()) + Imm._save_installed_checksums() + self.assertFalse(Imm._get_modules_with_changed_checksum()) + + +@common.at_install(False) +@common.post_install(True) +class TestModuleAfterInstall(TransactionCase): + + def setUp(self): + super(TestModuleAfterInstall, self).setUp() + Imm = self.env['ir.module.module'] + self.own_module = Imm.search([('name', '=', MODULE_NAME)]) + self.base_module = Imm.search([('name', '=', 'base')]) + + def test_get_modules_partially_installed(self): + Imm = self.env['ir.module.module'] + self.assertTrue( + self.own_module not in Imm._get_modules_partially_installed()) + self.own_module.button_upgrade() + self.assertTrue( + self.own_module in Imm._get_modules_partially_installed()) + self.own_module.button_upgrade_cancel() + self.assertTrue( + self.own_module not in Imm._get_modules_partially_installed()) + + def test_upgrade_changed_checksum(self): + Imm = self.env['ir.module.module'] + Bmu = self.env['base.module.upgrade'] + + # check modules are in installed state + installed_modules = Imm.search([('state', '=', 'installed')]) + self.assertTrue(self.own_module in installed_modules) + self.assertTrue(self.base_module in installed_modules) + self.assertTrue(len(installed_modules) > 2) + # change the checksum of 'base' + Imm._save_installed_checksums() + saved_checksums = Imm._get_saved_checksums() + saved_checksums['base'] = False + Imm._save_checksums(saved_checksums) + changed_modules = Imm._get_modules_with_changed_checksum() + self.assertEqual(len(changed_modules), 1) + self.assertTrue(self.base_module in changed_modules) + + def upgrade_module_mock(self_model): + upgrade_module_mock.call_count += 1 + # since we are upgrading base, all installed module + # must have been marked to upgrade at this stage + self.assertEqual(self.base_module.state, 'to upgrade') + self.assertEqual(self.own_module.state, 'to upgrade') + installed_modules.write({'state': 'installed'}) + + upgrade_module_mock.call_count = 0 + + # upgrade_changed_checksum commits, so mock that + with mock.patch.object(self.env.cr, 'commit'): + + # we simulate an install by setting module states + Bmu._patch_method('upgrade_module', upgrade_module_mock) + try: + Imm.upgrade_changed_checksum() + self.assertEqual(upgrade_module_mock.call_count, 1) + self.assertEqual(self.base_module.state, 'installed') + self.assertEqual(self.own_module.state, 'installed') + saved_checksums = Imm._get_saved_checksums() + self.assertTrue(saved_checksums['base']) + self.assertTrue(saved_checksums[MODULE_NAME]) + finally: + Bmu._revert_method('upgrade_module') + + def test_incomplete_upgrade(self): + Imm = self.env['ir.module.module'] + Bmu = self.env['base.module.upgrade'] + + installed_modules = Imm.search([('state', '=', 'installed')]) + # change the checksum of 'base' + Imm._save_installed_checksums() + saved_checksums = Imm._get_saved_checksums() + saved_checksums['base'] = False + Imm._save_checksums(saved_checksums) + + def upgrade_module_mock(self_model): + upgrade_module_mock.call_count += 1 + # since we are upgrading base, all installed module + # must have been marked to upgrade at this stage + self.assertEqual(self.base_module.state, 'to upgrade') + self.assertEqual(self.own_module.state, 'to upgrade') + installed_modules.write({'state': 'installed'}) + # simulate partial upgrade + self.own_module.write({'state': 'to upgrade'}) + + upgrade_module_mock.call_count = 0 + + # upgrade_changed_checksum commits, so mock that + with mock.patch.object(self.env.cr, 'commit'): + + # we simulate an install by setting module states + Bmu._patch_method('upgrade_module', upgrade_module_mock) + try: + with self.assertRaises(IncompleteUpgradeError): + Imm.upgrade_changed_checksum() + self.assertEqual(upgrade_module_mock.call_count, 1) + finally: + Bmu._revert_method('upgrade_module') + + def test_nothing_to_upgrade(self): + Imm = self.env['ir.module.module'] + Bmu = self.env['base.module.upgrade'] + + Imm._save_installed_checksums() + + def upgrade_module_mock(self_model): + upgrade_module_mock.call_count += 1 + + upgrade_module_mock.call_count = 0 + + # upgrade_changed_checksum commits, so mock that + with mock.patch.object(self.env.cr, 'commit'): + + # we simulate an install by setting module states + Bmu._patch_method('upgrade_module', upgrade_module_mock) + try: + Imm.upgrade_changed_checksum() + self.assertEqual(upgrade_module_mock.call_count, 0) + finally: + Bmu._revert_method('upgrade_module') diff --git a/module_auto_update/tests/test_module_deprecated.py b/module_auto_update/tests/test_module_deprecated.py index 51f347677..b82fd6748 100644 --- a/module_auto_update/tests/test_module_deprecated.py +++ b/module_auto_update/tests/test_module_deprecated.py @@ -1,7 +1,6 @@ # Copyright 2017 LasLabs Inc. # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). -import logging import os import tempfile @@ -11,13 +10,10 @@ from odoo.modules import get_module_path from odoo.tests.common import TransactionCase from odoo.tools import mute_logger +from ..addon_hash import addon_hash + from .. import post_init_hook -_logger = logging.getLogger(__name__) -try: - from checksumdir import dirhash -except ImportError: - _logger.debug('Cannot `import checksumdir`.') model = 'odoo.addons.module_auto_update.models.module_deprecated' @@ -35,10 +31,11 @@ class TestModule(TransactionCase): ('name', '=', module_name), ]) self.own_dir_path = get_module_path(module_name) - self.own_checksum = dirhash( + keep_langs = self.env['res.lang'].search([]).mapped('code') + self.own_checksum = addon_hash( self.own_dir_path, - 'sha1', - excluded_extensions=['pyc', 'pyo'], + exclude_patterns=['*.pyc', '*.pyo', '*.pot', 'static/*'], + keep_langs=keep_langs, ) self.own_writeable = os.access(self.own_dir_path, os.W_OK) From ca6e1a39de960d2f565b60a12af708aff8b91287 Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Fri, 16 Mar 2018 12:28:20 +0000 Subject: [PATCH 3/7] [REF] module_auto_update: Step 3, backwards compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation of this addon proved being extremely buggy: - It supplied out of the box a enabled cron to update Odoo that didn't restart the server, which possibly meant that upgrades broke things. - It overloaded standard Odoo upgrade methods that made i.e. installing an addon sometimes forced to upgrade all other addons in the database. - The checksum system wasn't smart enough, and some files that didn't need a module upgrade triggered the upgrade. - It was based on a dirhash library that was untested. - Some updates were not detected properly. - Storing a column into `ir.module.module` sometimes forbids uninstalling the addon. Thanks to Stéphane Bidoul (ACSONE), now we have new methods to perform the same work in a safer and more stable way. All I'm doing here is: - Cron is disabled by default. - Installed checksums are no longer saved at first install. - Old installations should keep most functionality intact thanks to the migration script. - Drop some duplicated tests. - Allow module uninstallation by pre-removing the fields from ir.mode.model. - When uninstalling the addon, the deprecated features will get removed for next installs always. Besides that, fixes for the new implementation too: - When uninstalling the addon, we remove the stored checksum data, so further installations work as if the addon was installed from scratch. --- module_auto_update/README.rst | 21 +++++ module_auto_update/__init__.py | 2 +- module_auto_update/__manifest__.py | 4 +- .../data/cron_data_deprecated.xml | 2 +- module_auto_update/hooks.py | 12 ++- .../migrations/10.0.2.0.0/pre-migrate.py | 23 +++++ .../models/module_deprecated.py | 13 ++- .../tests/test_module_deprecated.py | 52 +---------- .../tests/test_module_upgrade_deprecated.py | 3 + .../wizards/module_upgrade_deprecated.py | 93 +++++++++++++------ 10 files changed, 139 insertions(+), 86 deletions(-) create mode 100644 module_auto_update/migrations/10.0.2.0.0/pre-migrate.py diff --git a/module_auto_update/README.rst b/module_auto_update/README.rst index 8989df79a..1dd66fbbd 100644 --- a/module_auto_update/README.rst +++ b/module_auto_update/README.rst @@ -48,6 +48,27 @@ in an Odoo shell session:: :alt: Try me on Runbot :target: https://runbot.odoo-community.org/runbot/149/11.0 +Known issues / Roadmap +====================== + +* Since version ``2.0.0``, some features have been deprecated. + When you upgrade from previous versions, these features will be kept for + backwards compatibility, but beware! They are buggy! + + If you install this addon from scratch, these features are disabled by + default. + + To force enabling or disabling the deprecated features, set a configuration + parameter called ``module_auto_update.enable_deprecated`` to either ``1`` + or ``0``. It is recommended that you disable them. + + Keep in mind that from this version, all upgrades are assumed to run in a + separate odoo instance, dedicated exclusively to upgrade Odoo. + +* When migrating the addon to new versions, the deprecated features should be + removed. To make it simple all deprecated features are found in files + suffixed with ``_deprecated``. + Bug Tracker =========== diff --git a/module_auto_update/__init__.py b/module_auto_update/__init__.py index c80c51237..a0c82fd06 100644 --- a/module_auto_update/__init__.py +++ b/module_auto_update/__init__.py @@ -2,4 +2,4 @@ from . import models from . import wizards -from .hooks import post_init_hook +from .hooks import uninstall_hook diff --git a/module_auto_update/__manifest__.py b/module_auto_update/__manifest__.py index 4f4375fc7..d39962c1c 100644 --- a/module_auto_update/__manifest__.py +++ b/module_auto_update/__manifest__.py @@ -6,7 +6,7 @@ 'summary': 'Automatically update Odoo modules', 'version': '11.0.1.0.0', 'category': 'Extra Tools', - 'website': 'https://odoo-community.org/', + 'website': 'https://github.com/OCA/server-tools', 'author': 'LasLabs, ' 'Juan José Scarafía, ' 'Tecnativa, ' @@ -15,7 +15,7 @@ 'license': 'LGPL-3', 'application': False, 'installable': True, - 'post_init_hook': 'post_init_hook', + 'uninstall_hook': 'uninstall_hook', 'depends': [ 'base', ], diff --git a/module_auto_update/data/cron_data_deprecated.xml b/module_auto_update/data/cron_data_deprecated.xml index 9b5b1cc6b..d903dbda6 100644 --- a/module_auto_update/data/cron_data_deprecated.xml +++ b/module_auto_update/data/cron_data_deprecated.xml @@ -6,7 +6,7 @@ Perform Module Upgrades - + 1 days diff --git a/module_auto_update/hooks.py b/module_auto_update/hooks.py index 396f78707..cd161a246 100644 --- a/module_auto_update/hooks.py +++ b/module_auto_update/hooks.py @@ -3,7 +3,15 @@ from odoo import SUPERUSER_ID, api +from .models.module import PARAM_INSTALLED_CHECKSUMS +from .models.module_deprecated import PARAM_DEPRECATED -def post_init_hook(cr, registry): + +def uninstall_hook(cr, registry): env = api.Environment(cr, SUPERUSER_ID, {}) - env['ir.module.module']._save_installed_checksums() + env["ir.config_parameter"].set_param(PARAM_INSTALLED_CHECKSUMS, False) + # TODO Remove from here when removing deprecated features + env["ir.config_parameter"].set_param(PARAM_DEPRECATED, False) + prefix = "module_auto_update.field_ir_module_module_checksum_%s" + fields = env.ref(prefix % "dir") | env.ref(prefix % "installed") + fields.with_context(_force_unlink=True).unlink() diff --git a/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py b/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py new file mode 100644 index 000000000..4fe36ede7 --- /dev/null +++ b/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright 2018 Tecnativa - Jairo Llopis +# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl). +import logging +from psycopg2 import IntegrityError +from openerp.addons.module_auto_update.models.module_deprecated import \ + PARAM_DEPRECATED + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + """Autoenable deprecated behavior.""" + try: + cr.execute( + "INSERT INTO ir_config_parameter (key, value) VALUES (%s, '1')", + (PARAM_DEPRECATED,) + ) + _logger.warn("Deprecated features have been autoenabled, see " + "addon's README to know how to upgrade to the new " + "supported autoupdate mechanism.") + except IntegrityError: + _logger.info("Deprecated features setting exists, not autoenabling") diff --git a/module_auto_update/models/module_deprecated.py b/module_auto_update/models/module_deprecated.py index 5f1f5b8e2..0cb9defaa 100644 --- a/module_auto_update/models/module_deprecated.py +++ b/module_auto_update/models/module_deprecated.py @@ -3,14 +3,18 @@ from odoo import api, fields, models +PARAM_DEPRECATED = "module_auto_update.enable_deprecated" + class Module(models.Model): _inherit = 'ir.module.module' checksum_dir = fields.Char( + deprecated=True, compute='_compute_checksum_dir', ) checksum_installed = fields.Char( + deprecated=True, compute='_compute_checksum_installed', inverse='_inverse_checksum_installed', store=False, @@ -27,14 +31,17 @@ class Module(models.Model): rec.checksum_installed = saved_checksums.get(rec.name, False) def _inverse_checksum_installed(self): - saved_checksums = self._get_saved_checksums() + checksums = self._get_saved_checksums() for rec in self: - saved_checksums[rec.name] = rec.checksum_installed - self._save_installed_checksums() + checksums[rec.name] = rec.checksum_installed + self._save_checksums(checksums) @api.multi def _store_checksum_installed(self, vals): """Store the right installed checksum, if addon is installed.""" + if not self.env["base.module.upgrade"]._autoupdate_deprecated(): + # Skip if deprecated features are disabled + return if 'checksum_installed' not in vals: try: version = vals["latest_version"] diff --git a/module_auto_update/tests/test_module_deprecated.py b/module_auto_update/tests/test_module_deprecated.py index b82fd6748..99d0ee4ec 100644 --- a/module_auto_update/tests/test_module_deprecated.py +++ b/module_auto_update/tests/test_module_deprecated.py @@ -2,7 +2,6 @@ # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). import os -import tempfile import mock @@ -10,12 +9,12 @@ from odoo.modules import get_module_path from odoo.tests.common import TransactionCase from odoo.tools import mute_logger -from ..addon_hash import addon_hash +from openerp.addons.module_auto_update.addon_hash import addon_hash -from .. import post_init_hook +from ..models.module_deprecated import PARAM_DEPRECATED -model = 'odoo.addons.module_auto_update.models.module_deprecated' +model = 'odoo.addons.module_auto_update.models.module' class EndTestException(Exception): @@ -27,6 +26,7 @@ class TestModule(TransactionCase): def setUp(self): super(TestModule, self).setUp() module_name = 'module_auto_update' + self.env["ir.config_parameter"].set_param(PARAM_DEPRECATED, "1") self.own_module = self.env['ir.module.module'].search([ ('name', '=', module_name), ]) @@ -45,37 +45,6 @@ class TestModule(TransactionCase): test_module = self.env['ir.module.module'].create(vals) return test_module - def test_compute_checksum_dir(self): - """It should compute the directory's SHA-1 hash""" - self.assertEqual( - self.own_module.checksum_dir, self.own_checksum, - 'Module directory checksum not computed properly', - ) - - def test_compute_checksum_dir_ignore_excluded(self): - """It should exclude .pyc/.pyo extensions from checksum - calculations""" - if not self.own_writeable: - self.skipTest("Own directory not writeable") - with tempfile.NamedTemporaryFile( - suffix='.pyc', dir=self.own_dir_path): - self.assertEqual( - self.own_module.checksum_dir, self.own_checksum, - 'SHA1 checksum does not ignore excluded extensions', - ) - - def test_compute_checksum_dir_recomputes_when_file_added(self): - """It should return a different value when a non-.pyc/.pyo file is - added to the module directory""" - if not self.own_writeable: - self.skipTest("Own directory not writeable") - with tempfile.NamedTemporaryFile( - suffix='.py', dir=self.own_dir_path): - self.assertNotEqual( - self.own_module.checksum_dir, self.own_checksum, - 'SHA1 checksum not recomputed', - ) - def test_store_checksum_installed_state_installed(self): """It should set the module's checksum_installed equal to checksum_dir when vals contain a ``latest_version`` str.""" @@ -239,16 +208,3 @@ class TestModule(TransactionCase): self.env['ir.module.module']._revert_method( '_store_checksum_installed', ) - - def test_post_init_hook(self): - """It should set checksum_installed equal to checksum_dir for all - installed modules""" - installed_modules = self.env['ir.module.module'].search([ - ('state', '=', 'installed'), - ]) - post_init_hook(self.env.cr, None) - self.assertListEqual( - installed_modules.mapped('checksum_dir'), - installed_modules.mapped('checksum_installed'), - 'Installed modules did not have checksum_installed stored', - ) diff --git a/module_auto_update/tests/test_module_upgrade_deprecated.py b/module_auto_update/tests/test_module_upgrade_deprecated.py index 880c80d00..13e729594 100644 --- a/module_auto_update/tests/test_module_upgrade_deprecated.py +++ b/module_auto_update/tests/test_module_upgrade_deprecated.py @@ -7,12 +7,15 @@ from odoo.modules import get_module_path from odoo.modules.registry import Registry from odoo.tests.common import TransactionCase +from ..models.module_deprecated import PARAM_DEPRECATED + class TestModuleUpgrade(TransactionCase): def setUp(self): super(TestModuleUpgrade, self).setUp() module_name = 'module_auto_update' + self.env["ir.config_parameter"].set_param(PARAM_DEPRECATED, "1") self.own_module = self.env['ir.module.module'].search([ ('name', '=', module_name), ]) diff --git a/module_auto_update/wizards/module_upgrade_deprecated.py b/module_auto_update/wizards/module_upgrade_deprecated.py index 8634c38e5..b44ec684d 100644 --- a/module_auto_update/wizards/module_upgrade_deprecated.py +++ b/module_auto_update/wizards/module_upgrade_deprecated.py @@ -1,49 +1,84 @@ # Copyright 2017 LasLabs Inc. # License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). +import logging + from odoo import api, models +from ..models.module_deprecated import PARAM_DEPRECATED + +_logger = logging.getLogger(__name__) + class ModuleUpgrade(models.TransientModel): _inherit = 'base.module.upgrade' @api.model - @api.returns('ir.module.module') + def _autoupdate_deprecated(self): + """Know if we should enable deprecated features.""" + deprecated = ( + self.env["ir.config_parameter"].get_param(PARAM_DEPRECATED)) + if deprecated is False: + # Enable deprecated features if this is the 1st automated update + # after the version that deprecated them (X.Y.2.0.0) + own_module = self.env["ir.module.module"].search([ + ("name", "=", "module_auto_update"), + ]) + try: + if own_module.latest_version.split(".")[2] == "1": + deprecated = "1" + except AttributeError: + pass # 1st install, there's no latest_version + return deprecated == "1" + + @api.model def get_module_list(self): """Set modules to upgrade searching by their dir checksum.""" - Module = self.env["ir.module.module"] - installed_modules = Module.search([('state', '=', 'installed')]) - upgradeable_modules = installed_modules.filtered( - lambda r: r.checksum_dir != r.checksum_installed, - ) - upgradeable_modules.button_upgrade() + if self._autoupdate_deprecated(): + Module = self.env["ir.module.module"] + installed_modules = Module.search([('state', '=', 'installed')]) + upgradeable_modules = installed_modules.filtered( + lambda r: r.checksum_dir != r.checksum_installed, + ) + upgradeable_modules.button_upgrade() return super(ModuleUpgrade, self).get_module_list() @api.multi def upgrade_module(self): """Make a fully automated addon upgrade.""" - # Compute updates by checksum when called in @api.model fashion - if not self: - self.get_module_list() - Module = self.env["ir.module.module"] - # Get every addon state before updating - pre_states = {addon["name"]: addon["state"] - for addon in Module.search_read([], ["name", "state"])} + if self._autoupdate_deprecated(): + _logger.warning( + "You are possibly using an unsupported upgrade system; " + "set '%s' system parameter to '0' and start calling " + "`env['ir.module.module'].upgrade_changed_checksum()` from " + "now on to get rid of this message. See module's README's " + "Known Issues section for further information on the matter." + ) + # Compute updates by checksum when called in @api.model fashion + self.env.cr.autocommit(True) # Avoid transaction lock + if not self: + self.get_module_list() + Module = self.env["ir.module.module"] + # Get every addon state before updating + pre_states = {addon["name"]: addon["state"] for addon + in Module.search_read([], ["name", "state"])} # Perform upgrades, possibly in a limited graph that excludes me result = super(ModuleUpgrade, self).upgrade_module() - # Reload environments, anything may have changed - self.env.clear() - # Update addons checksum if state changed and I wasn't uninstalled - own = Module.search_read( - [("name", "=", "module_auto_update")], - ["state"], - limit=1) - if own and own[0]["state"] != "uninstalled": - for addon in Module.search([]): - if addon.state != pre_states.get(addon.name): - # Trigger the write hook that should have been - # triggered when the module was [un]installed/updated in - # the limited module graph inside above call to super(), - # and updates its dir checksum as needed - addon.latest_version = addon.latest_version + if self._autoupdate_deprecated(): + self.env.cr.autocommit(False) + # Reload environments, anything may have changed + self.env.clear() + # Update addons checksum if state changed and I wasn't uninstalled + own = Module.search_read( + [("name", "=", "module_auto_update")], + ["state"], + limit=1) + if own and own[0]["state"] != "uninstalled": + for addon in Module.search([]): + if addon.state != pre_states.get(addon.name): + # Trigger the write hook that should have been + # triggered when the module was [un]installed/updated + # in the limited module graph inside above call to + # super(), and updates its dir checksum as needed + addon.latest_version = addon.latest_version return result From 75a50697b1be0babc87d404ed62b101a90a7a29c Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Tue, 27 Mar 2018 09:53:33 +0100 Subject: [PATCH 4/7] [FIX] module_auto_update: Add .pyo sample file (#1205) Without this patch, if your tests are run under a `PYTHONOPTIMIZE=2` precompiled environment, they'd fail with this error because a new `.pyo` file would be created.: FAIL: test_basic (openerp.addons.module_auto_update.tests.test_addon_hash.TestAddonHash) Traceback (most recent call last): ` File "/opt/odoo/auto/addons/module_auto_update/tests/test_addon_hash.py", line 41, in test_basic ` 'static/src/some.js', ` AssertionError: Lists differ: ['README.rst', 'data/f1.xml', ... != ['README.rst', 'data/f1.xml', ... ` ` First differing element 13: ` models/stuff.pyo ` static/src/some.js ` ` First list contains 1 additional elements. ` First extra element 14: ` static/src/some.js ` ` ['README.rst', ` 'data/f1.xml', ` 'data/f2.xml', ` 'i18n/en.po', ` 'i18n/en_US.po', ` 'i18n/fr.po', ` 'i18n/fr_BE.po', ` 'i18n/test.pot', ` 'i18n_extra/en.po', ` 'i18n_extra/fr.po', ` 'i18n_extra/nl_NL.po', ` 'models/stuff.py', ` 'models/stuff.pyc', ` - 'models/stuff.pyo', ` 'static/src/some.js'] Ran 3 tests in 0.005s FAILED With this patch, the `.pyo` file is included, so tests will pass anywhere. --- .../tests/sample_module/models/stuff.pyo | Bin 0 -> 109 bytes module_auto_update/tests/test_addon_hash.py | 1 + 2 files changed, 1 insertion(+) create mode 100644 module_auto_update/tests/sample_module/models/stuff.pyo diff --git a/module_auto_update/tests/sample_module/models/stuff.pyo b/module_auto_update/tests/sample_module/models/stuff.pyo new file mode 100644 index 0000000000000000000000000000000000000000..b592f19841e2fdfc90a584eb75703c9eed4782f2 GIT binary patch literal 109 zcmZSn%*$2zb3;@z0~9a;X$K%K<^U2YObm=Ej10jV%s@^iBaraR1S!w}Vsrwmp}3?p ZElsbWvIL~tCO1E&G$+*#q^}sH0{}a24`BcR literal 0 HcmV?d00001 diff --git a/module_auto_update/tests/test_addon_hash.py b/module_auto_update/tests/test_addon_hash.py index 7523b8ec4..3827c5aed 100644 --- a/module_auto_update/tests/test_addon_hash.py +++ b/module_auto_update/tests/test_addon_hash.py @@ -38,6 +38,7 @@ class TestAddonHash(unittest.TestCase): 'i18n_extra/nl_NL.po', 'models/stuff.py', 'models/stuff.pyc', + 'models/stuff.pyo', 'static/src/some.js', ]) From 61ab94e2b9328b40c310675c7d8f3b926fe65d8e Mon Sep 17 00:00:00 2001 From: "Laurent Mignon (ACSONE)" Date: Tue, 27 Mar 2018 14:45:58 +0200 Subject: [PATCH 5/7] [FIX] Forward port module_auto_update refactoring from 9.0 --- module_auto_update/migrations/10.0.2.0.0/pre-migrate.py | 2 +- module_auto_update/models/module.py | 4 ++-- module_auto_update/tests/test_module.py | 6 +++--- module_auto_update/tests/test_module_deprecated.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py b/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py index 4fe36ede7..b5f5aa86d 100644 --- a/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py +++ b/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py @@ -3,7 +3,7 @@ # License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl). import logging from psycopg2 import IntegrityError -from openerp.addons.module_auto_update.models.module_deprecated import \ +from odoo.addons.module_auto_update.models.module_deprecated import \ PARAM_DEPRECATED _logger = logging.getLogger(__name__) diff --git a/module_auto_update/models/module.py b/module_auto_update/models/module.py index 334ec9d16..6be1562de 100644 --- a/module_auto_update/models/module.py +++ b/module_auto_update/models/module.py @@ -7,8 +7,8 @@ import json import logging import os -from openerp import api, exceptions, models, tools -from openerp.modules.module import get_module_path +from odoo import api, exceptions, models, tools +from odoo.modules.module import get_module_path from ..addon_hash import addon_hash diff --git a/module_auto_update/tests/test_module.py b/module_auto_update/tests/test_module.py index b4f0746c3..6558f0276 100644 --- a/module_auto_update/tests/test_module.py +++ b/module_auto_update/tests/test_module.py @@ -8,9 +8,9 @@ import tempfile import mock -from openerp.modules import get_module_path -from openerp.tests import common -from openerp.tests.common import TransactionCase +from odoo.modules import get_module_path +from odoo.tests import common +from odoo.tests.common import TransactionCase from ..addon_hash import addon_hash from ..models.module import IncompleteUpgradeError, DEFAULT_EXCLUDE_PATTERNS diff --git a/module_auto_update/tests/test_module_deprecated.py b/module_auto_update/tests/test_module_deprecated.py index 99d0ee4ec..a83881d11 100644 --- a/module_auto_update/tests/test_module_deprecated.py +++ b/module_auto_update/tests/test_module_deprecated.py @@ -9,7 +9,7 @@ from odoo.modules import get_module_path from odoo.tests.common import TransactionCase from odoo.tools import mute_logger -from openerp.addons.module_auto_update.addon_hash import addon_hash +from odoo.addons.module_auto_update.addon_hash import addon_hash from ..models.module_deprecated import PARAM_DEPRECATED From 38891f7462fcfc0fe2d226feeaf02a77efbbac1c Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Tue, 27 Mar 2018 09:45:57 +0100 Subject: [PATCH 6/7] [FIX] module_auto_update: Rollback cursor if param exists Without this patch, when upgrading after you have stored the deprecated features parameter, the cursor became broken and no more migrations could happen. You got this error: Traceback (most recent call last): File "/usr/local/bin/odoo", line 6, in exec(compile(open(__file__).read(), __file__, 'exec')) File "/opt/odoo/custom/src/odoo/odoo.py", line 160, in main() File "/opt/odoo/custom/src/odoo/odoo.py", line 157, in main openerp.cli.main() File "/opt/odoo/custom/src/odoo/openerp/cli/command.py", line 64, in main o.run(args) File "/opt/odoo/custom/src/odoo/openerp/cli/shell.py", line 65, in run self.shell(openerp.tools.config['db_name']) File "/opt/odoo/custom/src/odoo/openerp/cli/shell.py", line 52, in shell registry = openerp.modules.registry.RegistryManager.get(dbname) File "/opt/odoo/custom/src/odoo/openerp/modules/registry.py", line 355, in get update_module) File "/opt/odoo/custom/src/odoo/openerp/modules/registry.py", line 386, in new openerp.modules.load_modules(registry._db, force_demo, status, update_module) File "/opt/odoo/custom/src/odoo/openerp/modules/loading.py", line 335, in load_modules force, status, report, loaded_modules, update_module) File "/opt/odoo/custom/src/odoo/openerp/modules/loading.py", line 239, in load_marked_modules loaded, processed = load_module_graph(cr, graph, progressdict, report=report, skip_modules=loaded_modules, perform_checks=perform_checks) File "/opt/odoo/custom/src/odoo/openerp/modules/loading.py", line 136, in load_module_graph registry.setup_models(cr, partial=True) File "/opt/odoo/custom/src/odoo/openerp/modules/registry.py", line 186, in setup_models cr.execute('select model, transient from ir_model where state=%s', ('manual',)) File "/opt/odoo/custom/src/odoo/openerp/sql_db.py", line 154, in wrapper return f(self, *args, **kwargs) File "/opt/odoo/custom/src/odoo/openerp/sql_db.py", line 233, in execute res = self._obj.execute(query, params) psycopg2.InternalError: current transaction is aborted, commands ignored until end of transaction block Now you can safely migrate, be that parameter pre-created or not. --- .../migrations/10.0.2.0.0/pre-migrate.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py b/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py index b5f5aa86d..92135d174 100644 --- a/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py +++ b/module_auto_update/migrations/10.0.2.0.0/pre-migrate.py @@ -12,10 +12,12 @@ _logger = logging.getLogger(__name__) def migrate(cr, version): """Autoenable deprecated behavior.""" try: - cr.execute( - "INSERT INTO ir_config_parameter (key, value) VALUES (%s, '1')", - (PARAM_DEPRECATED,) - ) + with cr.savepoint(): + cr.execute( + """INSERT INTO ir_config_parameter (key, value) + VALUES (%s, '1')""", + (PARAM_DEPRECATED,) + ) _logger.warn("Deprecated features have been autoenabled, see " "addon's README to know how to upgrade to the new " "supported autoupdate mechanism.") From 3d01c33146b723f716327122872eba6ac6140fdc Mon Sep 17 00:00:00 2001 From: Benjamin Willig Date: Mon, 9 Apr 2018 14:10:12 +0200 Subject: [PATCH 7/7] [CHG] updated version number --- module_auto_update/__manifest__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module_auto_update/__manifest__.py b/module_auto_update/__manifest__.py index d39962c1c..1fa4b8ae4 100644 --- a/module_auto_update/__manifest__.py +++ b/module_auto_update/__manifest__.py @@ -4,7 +4,7 @@ { 'name': 'Module Auto Update', 'summary': 'Automatically update Odoo modules', - 'version': '11.0.1.0.0', + 'version': '11.0.2.0.0', 'category': 'Extra Tools', 'website': 'https://github.com/OCA/server-tools', 'author': 'LasLabs, '