Browse Source
[REF] module_auto_update: Step 2, add new API
[REF] module_auto_update: Step 2, add new API
This code comes from the module_checksum_upgrade proposal at https://github.com/OCA/server-tools/pull/1176. * [ADD] module_checksum_upgrade It provides the core mechanism of module_auto_update without the cron nor any change to the standard upgrade mechanism. Instead it provides an API on which module_auto_update can build, as well as a method which can be called from a script to run the upgrade of modules for which the checksum has changed. * [IMP] refactor module_auto_update Make it depend on module_checksum_upgrade which provides the core mechanisms of managing the checksums. module_auto_update makes it automatic. * [IMP] module_checksum_upgrade: better exclusion mechanism Ignore files based on exclude patterns. Ignore uninstalled languages. Better default for patterns to ignore (*.pyc,*.pyo,*.pot,static/*) For better control on the hashing mechanism implement our own: it's quite easy, and the checksumdir module used previously had no test. * [MIG] module_auto_update: adapt to new checksum mechanism * [IMP] module_checksum_upgrade: raise in case of incomplete upgrade * [IMP] module_checksum_upgrade: improve default exclusion pattern * [IMP] module_checksum_upgrade: control translations overwrite * [IMP] module_checksum_upgrade: one more test * [IMP] module_checksum_upgrade: credits [ci skip]pull/1198/head
Stéphane Bidoul (ACSONE)
7 years ago
No known key found for this signature in database
GPG Key ID: BCAB2555446B5B92
25 changed files with 534 additions and 61 deletions
-
43module_auto_update/README.rst
-
8module_auto_update/__openerp__.py
-
45module_auto_update/addon_hash.py
-
6module_auto_update/hooks.py
-
1module_auto_update/models/__init__.py
-
148module_auto_update/models/module.py
-
44module_auto_update/models/module_deprecated.py
-
2module_auto_update/tests/__init__.py
-
1module_auto_update/tests/sample_module/README.rst
-
1module_auto_update/tests/sample_module/data/f1.xml
-
1module_auto_update/tests/sample_module/data/f2.xml
-
1module_auto_update/tests/sample_module/i18n/en.po
-
1module_auto_update/tests/sample_module/i18n/en_US.po
-
1module_auto_update/tests/sample_module/i18n/fr.po
-
1module_auto_update/tests/sample_module/i18n/fr_BE.po
-
1module_auto_update/tests/sample_module/i18n/test.pot
-
1module_auto_update/tests/sample_module/i18n_extra/en.po
-
1module_auto_update/tests/sample_module/i18n_extra/fr.po
-
1module_auto_update/tests/sample_module/i18n_extra/nl_NL.po
-
1module_auto_update/tests/sample_module/models/stuff.py
-
BINmodule_auto_update/tests/sample_module/models/stuff.pyc
-
1module_auto_update/tests/sample_module/static/src/some.js
-
67module_auto_update/tests/test_addon_hash.py
-
203module_auto_update/tests/test_module.py
-
15module_auto_update/tests/test_module_deprecated.py
@ -0,0 +1,45 @@ |
|||
# -*- coding: utf-8 -*- |
|||
# Copyright 2018 ACSONE SA/NV. |
|||
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). |
|||
|
|||
from fnmatch import fnmatch |
|||
import hashlib |
|||
import os |
|||
|
|||
|
|||
def _fnmatch(filename, patterns): |
|||
for pattern in patterns: |
|||
if fnmatch(filename, pattern): |
|||
return True |
|||
return False |
|||
|
|||
|
|||
def _walk(top, exclude_patterns, keep_langs): |
|||
keep_langs = {l.split('_')[0] for l in keep_langs} |
|||
for dirpath, dirnames, filenames in os.walk(top): |
|||
dirnames.sort() |
|||
reldir = os.path.relpath(dirpath, top) |
|||
if reldir == '.': |
|||
reldir = '' |
|||
for filename in sorted(filenames): |
|||
filepath = os.path.join(reldir, filename) |
|||
if _fnmatch(filepath, exclude_patterns): |
|||
continue |
|||
if keep_langs and reldir in {'i18n', 'i18n_extra'}: |
|||
basename, ext = os.path.splitext(filename) |
|||
if ext == '.po': |
|||
if basename.split('_')[0] not in keep_langs: |
|||
continue |
|||
yield filepath |
|||
|
|||
|
|||
def addon_hash(top, exclude_patterns, keep_langs): |
|||
"""Compute a sha1 digest of file contents.""" |
|||
m = hashlib.sha1() |
|||
for filepath in _walk(top, exclude_patterns, keep_langs): |
|||
# hash filename so empty files influence the hash |
|||
m.update(filepath.encode('utf-8')) |
|||
# hash file content |
|||
with open(os.path.join(top, filepath), 'rb') as f: |
|||
m.update(f.read()) |
|||
return m.hexdigest() |
@ -0,0 +1,148 @@ |
|||
# -*- coding: utf-8 -*- |
|||
# Copyright 2017 LasLabs Inc. |
|||
# Copyright 2018 ACSONE SA/NV. |
|||
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). |
|||
|
|||
import json |
|||
import logging |
|||
import os |
|||
|
|||
from openerp import api, exceptions, models, tools |
|||
from openerp.modules.module import get_module_path |
|||
|
|||
from ..addon_hash import addon_hash |
|||
|
|||
PARAM_INSTALLED_CHECKSUMS = \ |
|||
'module_auto_update.installed_checksums' |
|||
PARAM_EXCLUDE_PATTERNS = \ |
|||
'module_auto_update.exclude_patterns' |
|||
DEFAULT_EXCLUDE_PATTERNS = \ |
|||
'*.pyc,*.pyo,i18n/*.pot,i18n_extra/*.pot,static/*' |
|||
|
|||
_logger = logging.getLogger(__name__) |
|||
|
|||
|
|||
class IncompleteUpgradeError(exceptions.UserError): |
|||
pass |
|||
|
|||
|
|||
class Module(models.Model): |
|||
_inherit = 'ir.module.module' |
|||
|
|||
@api.multi |
|||
def _get_checksum_dir(self): |
|||
self.ensure_one() |
|||
|
|||
exclude_patterns = self.env["ir.config_parameter"].get_param( |
|||
PARAM_EXCLUDE_PATTERNS, |
|||
DEFAULT_EXCLUDE_PATTERNS, |
|||
) |
|||
exclude_patterns = [p.strip() for p in exclude_patterns.split(',')] |
|||
keep_langs = self.env['res.lang'].search([]).mapped('code') |
|||
|
|||
module_path = get_module_path(self.name) |
|||
if module_path and os.path.isdir(module_path): |
|||
checksum_dir = addon_hash( |
|||
module_path, |
|||
exclude_patterns, |
|||
keep_langs, |
|||
) |
|||
else: |
|||
checksum_dir = False |
|||
|
|||
return checksum_dir |
|||
|
|||
@api.model |
|||
def _get_saved_checksums(self): |
|||
Icp = self.env['ir.config_parameter'] |
|||
return json.loads(Icp.get_param(PARAM_INSTALLED_CHECKSUMS, '{}')) |
|||
|
|||
@api.model |
|||
def _save_checksums(self, checksums): |
|||
Icp = self.env['ir.config_parameter'] |
|||
Icp.set_param(PARAM_INSTALLED_CHECKSUMS, json.dumps(checksums)) |
|||
|
|||
@api.model |
|||
def _save_installed_checksums(self): |
|||
checksums = {} |
|||
installed_modules = self.search([('state', '=', 'installed')]) |
|||
for module in installed_modules: |
|||
checksums[module.name] = module._get_checksum_dir() |
|||
self._save_checksums(checksums) |
|||
|
|||
@api.model |
|||
def _get_modules_partially_installed(self): |
|||
return self.search([ |
|||
('state', 'in', ('to install', 'to remove', 'to upgrade')), |
|||
]) |
|||
|
|||
@api.model |
|||
def _get_modules_with_changed_checksum(self): |
|||
saved_checksums = self._get_saved_checksums() |
|||
installed_modules = self.search([('state', '=', 'installed')]) |
|||
return installed_modules.filtered( |
|||
lambda r: r._get_checksum_dir() != saved_checksums.get(r.name), |
|||
) |
|||
|
|||
@api.model |
|||
def upgrade_changed_checksum(self, overwrite_existing_translations=False): |
|||
"""Run an upgrade of the database, upgrading only changed modules. |
|||
|
|||
Installed modules for which the checksum has changed since the |
|||
last successful run of this method are marked "to upgrade", |
|||
then the normal Odoo scheduled upgrade process |
|||
is launched. |
|||
|
|||
If there is no module with a changed checksum, and no module in state |
|||
other than installed, uninstalled, uninstallable, this method does |
|||
nothing, otherwise the normal Odoo upgrade process is launched. |
|||
|
|||
After a successful upgrade, the checksums of installed modules are |
|||
saved. |
|||
|
|||
In case of error during the upgrade, an exception is raised. |
|||
If any module remains to upgrade or to uninstall after the upgrade |
|||
process, an exception is raised as well. |
|||
|
|||
Note: this method commits the current transaction at each important |
|||
step, it is therefore not intended to be run as part of a |
|||
larger transaction. |
|||
""" |
|||
_logger.info( |
|||
"Checksum upgrade starting (i18n-overwrite=%s)...", |
|||
overwrite_existing_translations |
|||
) |
|||
|
|||
tools.config['overwrite_existing_translations'] = \ |
|||
overwrite_existing_translations |
|||
|
|||
_logger.info("Updating modules list...") |
|||
self.update_list() |
|||
changed_modules = self._get_modules_with_changed_checksum() |
|||
if not changed_modules and not self._get_modules_partially_installed(): |
|||
_logger.info("No checksum change detected in installed modules " |
|||
"and all modules installed, nothing to do.") |
|||
return |
|||
_logger.info("Marking the following modules to upgrade, " |
|||
"for their checksums changed: %s...", |
|||
','.join(changed_modules.mapped('name'))) |
|||
changed_modules.button_upgrade() |
|||
self.env.cr.commit() # pylint: disable=invalid-commit |
|||
|
|||
_logger.info("Upgrading...") |
|||
self.env['base.module.upgrade'].upgrade_module() |
|||
self.env.cr.commit() # pylint: disable=invalid-commit |
|||
|
|||
_logger.info("Upgrade successful, updating checksums...") |
|||
self._save_installed_checksums() |
|||
self.env.cr.commit() # pylint: disable=invalid-commit |
|||
|
|||
partial_modules = self._get_modules_partially_installed() |
|||
if partial_modules: |
|||
raise IncompleteUpgradeError( |
|||
"Checksum upgrade successful " |
|||
"but incomplete for the following modules: %s" % |
|||
','.join(partial_modules.mapped('name')) |
|||
) |
|||
|
|||
_logger.info("Checksum upgrade complete.") |
@ -0,0 +1 @@ |
|||
Test data for addon_hash module. |
@ -0,0 +1 @@ |
|||
<odoo/> |
@ -0,0 +1 @@ |
|||
<odoo/> |
@ -0,0 +1 @@ |
|||
en text |
@ -0,0 +1 @@ |
|||
en_US |
@ -0,0 +1 @@ |
|||
fr |
@ -0,0 +1 @@ |
|||
fr_BE |
@ -0,0 +1 @@ |
|||
... |
@ -0,0 +1 @@ |
|||
en |
@ -0,0 +1 @@ |
|||
fr |
@ -0,0 +1 @@ |
|||
nl_NL |
@ -0,0 +1 @@ |
|||
1+1 |
@ -0,0 +1 @@ |
|||
/* javascript */ |
@ -0,0 +1,67 @@ |
|||
# -*- coding: utf-8 -*- |
|||
# Copyright 2018 ACSONE SA/NV. |
|||
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). |
|||
|
|||
import os |
|||
import unittest |
|||
|
|||
from .. import addon_hash |
|||
from ..models.module import DEFAULT_EXCLUDE_PATTERNS |
|||
|
|||
|
|||
class TestAddonHash(unittest.TestCase): |
|||
|
|||
def setUp(self): |
|||
super(TestAddonHash, self).setUp() |
|||
self.sample_dir = os.path.join( |
|||
os.path.dirname(__file__), |
|||
'sample_module', |
|||
) |
|||
|
|||
def test_basic(self): |
|||
files = list(addon_hash._walk( |
|||
self.sample_dir, |
|||
exclude_patterns=[], |
|||
keep_langs=[], |
|||
)) |
|||
self.assertEqual(files, [ |
|||
'README.rst', |
|||
'data/f1.xml', |
|||
'data/f2.xml', |
|||
'i18n/en.po', |
|||
'i18n/en_US.po', |
|||
'i18n/fr.po', |
|||
'i18n/fr_BE.po', |
|||
'i18n/test.pot', |
|||
'i18n_extra/en.po', |
|||
'i18n_extra/fr.po', |
|||
'i18n_extra/nl_NL.po', |
|||
'models/stuff.py', |
|||
'models/stuff.pyc', |
|||
'static/src/some.js', |
|||
]) |
|||
|
|||
def test_exclude(self): |
|||
files = list(addon_hash._walk( |
|||
self.sample_dir, |
|||
exclude_patterns=DEFAULT_EXCLUDE_PATTERNS.split(','), |
|||
keep_langs=['fr_FR', 'nl'], |
|||
)) |
|||
self.assertEqual(files, [ |
|||
'README.rst', |
|||
'data/f1.xml', |
|||
'data/f2.xml', |
|||
'i18n/fr.po', |
|||
'i18n/fr_BE.po', |
|||
'i18n_extra/fr.po', |
|||
'i18n_extra/nl_NL.po', |
|||
'models/stuff.py', |
|||
]) |
|||
|
|||
def test2(self): |
|||
checksum = addon_hash.addon_hash( |
|||
self.sample_dir, |
|||
exclude_patterns=['*.pyc', '*.pyo', '*.pot', 'static/*'], |
|||
keep_langs=['fr_FR', 'nl'], |
|||
) |
|||
self.assertEqual(checksum, 'fecb89486c8a29d1f760cbd01c1950f6e8421b14') |
@ -0,0 +1,203 @@ |
|||
# -*- coding: utf-8 -*- |
|||
# Copyright 2017 LasLabs Inc. |
|||
# Copyright 2018 ACSONE SA/NV. |
|||
# License LGPL-3.0 or later (http://www.gnu.org/licenses/lgpl). |
|||
|
|||
import os |
|||
import tempfile |
|||
|
|||
import mock |
|||
|
|||
from openerp.modules import get_module_path |
|||
from openerp.tests import common |
|||
from openerp.tests.common import TransactionCase |
|||
|
|||
from ..addon_hash import addon_hash |
|||
from ..models.module import IncompleteUpgradeError, DEFAULT_EXCLUDE_PATTERNS |
|||
|
|||
MODULE_NAME = 'module_auto_update' |
|||
|
|||
|
|||
class TestModule(TransactionCase): |
|||
|
|||
def setUp(self): |
|||
super(TestModule, self).setUp() |
|||
self.own_module = self.env['ir.module.module'].search([ |
|||
('name', '=', MODULE_NAME), |
|||
]) |
|||
self.own_dir_path = get_module_path(MODULE_NAME) |
|||
keep_langs = self.env['res.lang'].search([]).mapped('code') |
|||
self.own_checksum = addon_hash( |
|||
self.own_dir_path, |
|||
exclude_patterns=DEFAULT_EXCLUDE_PATTERNS.split(','), |
|||
keep_langs=keep_langs, |
|||
) |
|||
self.own_writeable = os.access(self.own_dir_path, os.W_OK) |
|||
|
|||
def test_compute_checksum_dir(self): |
|||
"""It should compute the directory's SHA-1 hash""" |
|||
self.assertEqual( |
|||
self.own_module._get_checksum_dir(), self.own_checksum, |
|||
'Module directory checksum not computed properly', |
|||
) |
|||
|
|||
def test_compute_checksum_dir_ignore_excluded(self): |
|||
"""It should exclude .pyc/.pyo extensions from checksum |
|||
calculations""" |
|||
if not self.own_writeable: |
|||
self.skipTest("Own directory not writeable") |
|||
with tempfile.NamedTemporaryFile(suffix='.pyc', dir=self.own_dir_path): |
|||
self.assertEqual( |
|||
self.own_module._get_checksum_dir(), self.own_checksum, |
|||
'SHA1 checksum does not ignore excluded extensions', |
|||
) |
|||
|
|||
def test_compute_checksum_dir_recomputes_when_file_added(self): |
|||
"""It should return a different value when a non-.pyc/.pyo file is |
|||
added to the module directory""" |
|||
if not self.own_writeable: |
|||
self.skipTest("Own directory not writeable") |
|||
with tempfile.NamedTemporaryFile(suffix='.py', dir=self.own_dir_path): |
|||
self.assertNotEqual( |
|||
self.own_module._get_checksum_dir(), self.own_checksum, |
|||
'SHA1 checksum not recomputed', |
|||
) |
|||
|
|||
def test_saved_checksums(self): |
|||
Imm = self.env['ir.module.module'] |
|||
base_module = Imm.search([('name', '=', 'base')]) |
|||
self.assertEqual(base_module.state, 'installed') |
|||
self.assertFalse(Imm._get_saved_checksums()) |
|||
Imm._save_installed_checksums() |
|||
saved_checksums = Imm._get_saved_checksums() |
|||
self.assertTrue(saved_checksums) |
|||
self.assertTrue(saved_checksums['base']) |
|||
|
|||
def test_get_modules_with_changed_checksum(self): |
|||
Imm = self.env['ir.module.module'] |
|||
self.assertTrue(Imm._get_modules_with_changed_checksum()) |
|||
Imm._save_installed_checksums() |
|||
self.assertFalse(Imm._get_modules_with_changed_checksum()) |
|||
|
|||
|
|||
@common.at_install(False) |
|||
@common.post_install(True) |
|||
class TestModuleAfterInstall(TransactionCase): |
|||
|
|||
def setUp(self): |
|||
super(TestModuleAfterInstall, self).setUp() |
|||
Imm = self.env['ir.module.module'] |
|||
self.own_module = Imm.search([('name', '=', MODULE_NAME)]) |
|||
self.base_module = Imm.search([('name', '=', 'base')]) |
|||
|
|||
def test_get_modules_partially_installed(self): |
|||
Imm = self.env['ir.module.module'] |
|||
self.assertTrue( |
|||
self.own_module not in Imm._get_modules_partially_installed()) |
|||
self.own_module.button_upgrade() |
|||
self.assertTrue( |
|||
self.own_module in Imm._get_modules_partially_installed()) |
|||
self.own_module.button_upgrade_cancel() |
|||
self.assertTrue( |
|||
self.own_module not in Imm._get_modules_partially_installed()) |
|||
|
|||
def test_upgrade_changed_checksum(self): |
|||
Imm = self.env['ir.module.module'] |
|||
Bmu = self.env['base.module.upgrade'] |
|||
|
|||
# check modules are in installed state |
|||
installed_modules = Imm.search([('state', '=', 'installed')]) |
|||
self.assertTrue(self.own_module in installed_modules) |
|||
self.assertTrue(self.base_module in installed_modules) |
|||
self.assertTrue(len(installed_modules) > 2) |
|||
# change the checksum of 'base' |
|||
Imm._save_installed_checksums() |
|||
saved_checksums = Imm._get_saved_checksums() |
|||
saved_checksums['base'] = False |
|||
Imm._save_checksums(saved_checksums) |
|||
changed_modules = Imm._get_modules_with_changed_checksum() |
|||
self.assertEqual(len(changed_modules), 1) |
|||
self.assertTrue(self.base_module in changed_modules) |
|||
|
|||
def upgrade_module_mock(self_model): |
|||
upgrade_module_mock.call_count += 1 |
|||
# since we are upgrading base, all installed module |
|||
# must have been marked to upgrade at this stage |
|||
self.assertEqual(self.base_module.state, 'to upgrade') |
|||
self.assertEqual(self.own_module.state, 'to upgrade') |
|||
installed_modules.write({'state': 'installed'}) |
|||
|
|||
upgrade_module_mock.call_count = 0 |
|||
|
|||
# upgrade_changed_checksum commits, so mock that |
|||
with mock.patch.object(self.env.cr, 'commit'): |
|||
|
|||
# we simulate an install by setting module states |
|||
Bmu._patch_method('upgrade_module', upgrade_module_mock) |
|||
try: |
|||
Imm.upgrade_changed_checksum() |
|||
self.assertEqual(upgrade_module_mock.call_count, 1) |
|||
self.assertEqual(self.base_module.state, 'installed') |
|||
self.assertEqual(self.own_module.state, 'installed') |
|||
saved_checksums = Imm._get_saved_checksums() |
|||
self.assertTrue(saved_checksums['base']) |
|||
self.assertTrue(saved_checksums[MODULE_NAME]) |
|||
finally: |
|||
Bmu._revert_method('upgrade_module') |
|||
|
|||
def test_incomplete_upgrade(self): |
|||
Imm = self.env['ir.module.module'] |
|||
Bmu = self.env['base.module.upgrade'] |
|||
|
|||
installed_modules = Imm.search([('state', '=', 'installed')]) |
|||
# change the checksum of 'base' |
|||
Imm._save_installed_checksums() |
|||
saved_checksums = Imm._get_saved_checksums() |
|||
saved_checksums['base'] = False |
|||
Imm._save_checksums(saved_checksums) |
|||
|
|||
def upgrade_module_mock(self_model): |
|||
upgrade_module_mock.call_count += 1 |
|||
# since we are upgrading base, all installed module |
|||
# must have been marked to upgrade at this stage |
|||
self.assertEqual(self.base_module.state, 'to upgrade') |
|||
self.assertEqual(self.own_module.state, 'to upgrade') |
|||
installed_modules.write({'state': 'installed'}) |
|||
# simulate partial upgrade |
|||
self.own_module.write({'state': 'to upgrade'}) |
|||
|
|||
upgrade_module_mock.call_count = 0 |
|||
|
|||
# upgrade_changed_checksum commits, so mock that |
|||
with mock.patch.object(self.env.cr, 'commit'): |
|||
|
|||
# we simulate an install by setting module states |
|||
Bmu._patch_method('upgrade_module', upgrade_module_mock) |
|||
try: |
|||
with self.assertRaises(IncompleteUpgradeError): |
|||
Imm.upgrade_changed_checksum() |
|||
self.assertEqual(upgrade_module_mock.call_count, 1) |
|||
finally: |
|||
Bmu._revert_method('upgrade_module') |
|||
|
|||
def test_nothing_to_upgrade(self): |
|||
Imm = self.env['ir.module.module'] |
|||
Bmu = self.env['base.module.upgrade'] |
|||
|
|||
Imm._save_installed_checksums() |
|||
|
|||
def upgrade_module_mock(self_model): |
|||
upgrade_module_mock.call_count += 1 |
|||
|
|||
upgrade_module_mock.call_count = 0 |
|||
|
|||
# upgrade_changed_checksum commits, so mock that |
|||
with mock.patch.object(self.env.cr, 'commit'): |
|||
|
|||
# we simulate an install by setting module states |
|||
Bmu._patch_method('upgrade_module', upgrade_module_mock) |
|||
try: |
|||
Imm.upgrade_changed_checksum() |
|||
self.assertEqual(upgrade_module_mock.call_count, 0) |
|||
finally: |
|||
Bmu._revert_method('upgrade_module') |
Write
Preview
Loading…
Cancel
Save
Reference in new issue