From 3312a7b3a4d4d153707e8f2a6735bfda802445c6 Mon Sep 17 00:00:00 2001 From: default Date: Sun, 24 Jan 2021 23:16:42 +0000 Subject: [PATCH] [UPD] copier update --- .copier-answers.yml | 2 +- .pre-commit-config.yaml | 27 ++- README.md | 2 +- devel.yaml | 32 ++-- tasks.py | 352 +++++++++++++++++++++++++++++++++++----- 5 files changed, 351 insertions(+), 64 deletions(-) diff --git a/.copier-answers.yml b/.copier-answers.yml index 2e1f75a..92c3c6e 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,5 +1,5 @@ # Changes here will be overwritten by Copier -_commit: v2.4.2 +_commit: v2.6.1 _src_path: gh:Tecnativa/doodba-copier-template backup_deletion: true backup_dst: null diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 48fcc84..503de4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,4 +1,6 @@ exclude: | + # NOT INSTALLABLE ADDONS + # END NOT INSTALLABLE ADDONS (?x) # Files and folders generated by bots, to avoid loops /static/description/index\.html$| @@ -17,6 +19,14 @@ repos: entry: found forbidden files; remove them language: fail files: "\\.rej$" + - repo: https://github.com/oca/maintainer-tools + rev: b9c963d + hooks: + # update the NOT INSTALLABLE ADDONS section above + - id: oca-update-pre-commit-excluded-addons + args: + - --addons-dir + - odoo/custom/src/private - repo: https://github.com/psf/black rev: 20.8b1 hooks: @@ -25,6 +35,8 @@ repos: rev: v2.7.2 hooks: - id: pyupgrade + args: + - --keep-percent-format - repo: https://github.com/timothycrosley/isort rev: 5.5.1 hooks: @@ -32,15 +44,14 @@ repos: name: isort except __init__.py args: [--settings, .] exclude: /__init__\.py$ - # TODO Migrate to new prettier pre-commit repo after fixed: - # HACK https://github.com/prettier/pre-commit/pull/17 - # HACK https://github.com/prettier/pre-commit/pull/18 - - repo: https://github.com/prettier/prettier - rev: 2.1.2 + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.1.2 hooks: - id: prettier name: prettier + plugin-xml additional_dependencies: + # HACK https://github.com/prettier/pre-commit/issues/16#issuecomment-713474520 + - prettier@2.1.2 - "@prettier/plugin-xml@0.12.0" args: - --plugin=@prettier/plugin-xml @@ -73,7 +84,7 @@ repos: files: /__init__\.py$ additional_dependencies: ["flake8-bugbear==20.1.4"] - repo: https://github.com/pycqa/pylint - rev: pylint-2.6.0 + rev: pylint-2.5.3 hooks: - id: pylint name: pylint with optional checks @@ -83,7 +94,7 @@ repos: - --exit-zero verbose: true additional_dependencies: - - isort==4.3.21 + - isort==4.3.4 - pylint-odoo==3.5.0 - id: pylint name: pylint with mandatory checks @@ -91,7 +102,7 @@ repos: - --valid_odoo_versions=14.0 - --rcfile=.pylintrc-mandatory additional_dependencies: - - isort==4.3.21 + - isort==4.3.4 - pylint-odoo==3.5.0 - repo: https://github.com/pre-commit/mirrors-eslint rev: v7.8.1 diff --git a/README.md b/README.md index 5e8c08d..75bffaa 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Doodba deployment](https://img.shields.io/badge/deployment-doodba-informational)](https://github.com/Tecnativa/doodba) -[![Last template update](https://img.shields.io/badge/last%20template%20update-v2.4.2-informational)](https://github.com/Tecnativa/doodba-copier-template/tree/v2.4.2) +[![Last template update](https://img.shields.io/badge/last%20template%20update-v2.6.1-informational)](https://github.com/Tecnativa/doodba-copier-template/tree/v2.6.1) [![Odoo](https://img.shields.io/badge/odoo-v14.0-a3478a)](https://github.com/odoo/odoo/tree/14.0) [![AGPL-3.0-or-later license](https://img.shields.io/badge/license-AGPL--3.0--or--later-success})](LICENSE) [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white)](https://pre-commit.com/) diff --git a/devel.yaml b/devel.yaml index 01d7a97..d4ac030 100644 --- a/devel.yaml +++ b/devel.yaml @@ -46,12 +46,13 @@ services: - ./odoo/custom:/opt/odoo/custom:ro,z - ./odoo/auto:/opt/odoo/auto:rw,z depends_on: - - cdnjs_cloudflare_proxy - db - - fonts_googleapis_proxy - - fonts_gstatic_proxy - - google_proxy - - gravatar_proxy + - proxy_cdnjs_cloudflare_com + - proxy_fonts_googleapis_com + - proxy_fonts_gstatic_com + - proxy_www_google_com + - proxy_www_googleapis_com + - proxy_www_gravatar_com - smtp - wdb command: @@ -97,7 +98,7 @@ services: init: true # Whitelist outgoing traffic for tests, reports, etc. - cdnjs_cloudflare_proxy: + proxy_cdnjs_cloudflare_com: image: tecnativa/whitelist networks: default: @@ -108,7 +109,7 @@ services: TARGET: cdnjs.cloudflare.com PRE_RESOLVE: 1 - fonts_googleapis_proxy: + proxy_fonts_googleapis_com: image: tecnativa/whitelist networks: default: @@ -119,7 +120,7 @@ services: TARGET: fonts.googleapis.com PRE_RESOLVE: 1 - fonts_gstatic_proxy: + proxy_fonts_gstatic_com: image: tecnativa/whitelist networks: default: @@ -130,7 +131,7 @@ services: TARGET: fonts.gstatic.com PRE_RESOLVE: 1 - google_proxy: + proxy_www_google_com: image: tecnativa/whitelist networks: default: @@ -141,7 +142,18 @@ services: TARGET: www.google.com PRE_RESOLVE: 1 - gravatar_proxy: + proxy_www_googleapis_com: + image: tecnativa/whitelist + networks: + default: + aliases: + - www.googleapis.com + public: + environment: + TARGET: www.googleapis.com + PRE_RESOLVE: 1 + + proxy_www_gravatar_com: image: tecnativa/whitelist networks: default: diff --git a/tasks.py b/tasks.py index 88617ab..4143ecd 100644 --- a/tasks.py +++ b/tasks.py @@ -6,16 +6,70 @@ Contains common helpers to develop using this child project. """ import json import os +import tempfile +import time from itertools import chain +from logging import getLogger from pathlib import Path from shutil import which -from invoke import task +from invoke import exceptions, task +from invoke.util import yaml -ODOO_VERSION = 14.0 PROJECT_ROOT = Path(__file__).parent.absolute() SRC_PATH = PROJECT_ROOT / "odoo" / "custom" / "src" UID_ENV = {"GID": str(os.getgid()), "UID": str(os.getuid()), "UMASK": "27"} +SERVICES_WAIT_TIME = int(os.environ.get("SERVICES_WAIT_TIME", 4)) +ODOO_VERSION = float( + yaml.safe_load((PROJECT_ROOT / "common.yaml").read_text())["services"]["odoo"][ + "build" + ]["args"]["ODOO_VERSION"] +) + +_logger = getLogger(__name__) + + +def _override_docker_command(service, command, file, orig_file=None): + # Read config from main file + if orig_file: + with open(orig_file, "r") as fd: + orig_docker_config = yaml.safe_load(fd.read()) + docker_compose_file_version = orig_docker_config.get("version") + else: + docker_compose_file_version = "2.4" + docker_config = { + "version": docker_compose_file_version, + "services": {service: {"command": command}}, + } + docker_config_yaml = yaml.dump(docker_config) + file.write(docker_config_yaml) + file.flush() + + +def _remove_auto_reload(file, orig_file): + with open(orig_file, "r") as fd: + orig_docker_config = yaml.safe_load(fd.read()) + odoo_command = orig_docker_config["services"]["odoo"]["command"] + new_odoo_command = [] + for flag in odoo_command: + if flag.startswith("--dev"): + flag = flag.replace("reload,", "") + new_odoo_command.append(flag) + _override_docker_command("odoo", new_odoo_command, file, orig_file=orig_file) + + +def _get_cwd_addon(file): + cwd = Path(file) + manifest_file = False + while PROJECT_ROOT < cwd: + manifest_file = (cwd / "__manifest__.py").exists() or ( + cwd / "__openerp__.py" + ).exists() + if manifest_file: + return cwd.stem + cwd = cwd.parent + if cwd == PROJECT_ROOT: + return None @task @@ -38,7 +92,7 @@ def write_code_workspace_file(c, cw_path=None): Example: `--cw-path doodba.my-custom-name.code-workspace` """ root_name = f"doodba.{PROJECT_ROOT.name}" - root_var = "${workspaceRoot:%s}" % root_name + root_var = "${workspaceFolder:%s}" % root_name if not cw_path: try: cw_path = next(PROJECT_ROOT.glob("doodba.*.code-workspace")) @@ -52,14 +106,18 @@ def write_code_workspace_file(c, cw_path=None): cw_config = json.load(cw_fd) except (FileNotFoundError, json.decoder.JSONDecodeError): pass # Nevermind, we start with a new config + # Static settings + cw_config.setdefault("settings", {}) + cw_config["settings"].update({"search.followSymlinks": False}) # Launch configurations debugpy_configuration = { "name": "Attach Python debugger to running container", "type": "python", "request": "attach", - "pathMappings": [{"localRoot": f"{root_var}/odoo", "remoteRoot": "/opt/odoo"}], + "pathMappings": [], "port": int(ODOO_VERSION) * 1000 + 899, - "host": "localhost", + # HACK https://github.com/microsoft/vscode-python/issues/14820 + "host": "0.0.0.0", } firefox_configuration = { "type": "firefox", @@ -93,30 +151,10 @@ def write_code_workspace_file(c, cw_path=None): "preLaunchTask": "Start Odoo in debug mode", }, { - "name": "Start Odoo and debug JS in Firefox", - "configurations": ["Connect to firefox debugger"], - "preLaunchTask": "Start Odoo", - }, - { - "name": "Start Odoo and debug JS in Chrome", - "configurations": ["Connect to chrome debugger"], - "preLaunchTask": "Start Odoo", - }, - { - "name": "Start Odoo and debug Python + JS in Firefox", - "configurations": [ - "Attach Python debugger to running container", - "Connect to firefox debugger", - ], - "preLaunchTask": "Start Odoo in debug mode", - }, - { - "name": "Start Odoo and debug Python + JS in Chrome", - "configurations": [ - "Attach Python debugger to running container", - "Connect to chrome debugger", - ], - "preLaunchTask": "Start Odoo in debug mode", + "name": "Test and debug current module", + "configurations": ["Attach Python debugger to running container"], + "preLaunchTask": "Run Odoo Tests in debug mode for current module", + "internalConsoleOptions": "openOnSessionStart", }, ], "configurations": [ @@ -125,7 +163,13 @@ def write_code_workspace_file(c, cw_path=None): chrome_configuration, ], } - # Configure folders + # Configure folders and debuggers + debugpy_configuration["pathMappings"].append( + { + "localRoot": "${workspaceFolder:odoo}/", + "remoteRoot": "/opt/odoo/custom/src/odoo", + } + ) cw_config["folders"] = [] for subrepo in SRC_PATH.glob("*"): if not subrepo.is_dir(): @@ -134,18 +178,25 @@ def write_code_workspace_file(c, cw_path=None): cw_config["folders"].append( {"path": str(subrepo.relative_to(PROJECT_ROOT))} ) - debugpy_configuration["pathMappings"].append( - { - "localRoot": "${workspaceRoot:%s}" % subrepo.name, - "remoteRoot": f"/opt/odoo/custom/src/{subrepo.name}", - } - ) for addon in chain(subrepo.glob("*"), subrepo.glob("addons/*")): if (addon / "__manifest__.py").is_file() or ( addon / "__openerp__.py" ).is_file(): + if subrepo.name == "odoo": + local_path = "${workspaceFolder:%s}/addons/%s/" % ( + subrepo.name, + addon.name, + ) + else: + local_path = "${workspaceFolder:%s}/%s" % (subrepo.name, addon.name) + debugpy_configuration["pathMappings"].append( + { + "localRoot": local_path, + "remoteRoot": f"/opt/odoo/auto/addons/{addon.name}/", + } + ) url = f"http://localhost:{ODOO_VERSION:.0f}069/{addon.name}/static/" - path = "${{workspaceRoot:{}}}/{}/static/".format( + path = "${workspaceFolder:%s}/%s/static/" % ( subrepo.name, addon.relative_to(subrepo), ) @@ -168,6 +219,44 @@ def write_code_workspace_file(c, cw_path=None): "clear": False, }, "problemMatcher": [], + "options": {"statusbar": {"label": "$(play-circle) Start Odoo"}}, + }, + { + "label": "Run Odoo Tests for current module", + "type": "process", + "command": "invoke", + "args": ["test", "--cur-file", "${file}"], + "presentation": { + "echo": True, + "reveal": "always", + "focus": True, + "panel": "shared", + "showReuseMessage": True, + "clear": False, + }, + "problemMatcher": [], + "options": {"statusbar": {"label": "$(beaker) Test module"}}, + }, + { + "label": "Run Odoo Tests in debug mode for current module", + "type": "process", + "command": "invoke", + "args": [ + "test", + "--cur-file", + "${file}", + "--debugpy", + ], + "presentation": { + "echo": True, + "reveal": "silent", + "focus": False, + "panel": "shared", + "showReuseMessage": True, + "clear": False, + }, + "problemMatcher": [], + "options": {"statusbar": {"hide": True}}, }, { "label": "Start Odoo in debug mode", @@ -183,6 +272,7 @@ def write_code_workspace_file(c, cw_path=None): "clear": False, }, "problemMatcher": [], + "options": {"statusbar": {"hide": True}}, }, { "label": "Stop Odoo", @@ -198,12 +288,29 @@ def write_code_workspace_file(c, cw_path=None): "clear": False, }, "problemMatcher": [], + "options": {"statusbar": {"label": "$(stop-circle) Stop Odoo"}}, + }, + { + "label": "Restart Odoo", + "type": "process", + "command": "invoke", + "args": ["restart"], + "presentation": { + "echo": True, + "reveal": "silent", + "focus": False, + "panel": "shared", + "showReuseMessage": True, + "clear": False, + }, + "problemMatcher": [], + "options": {"statusbar": {"label": "$(history) Restart Odoo"}}, }, ], } # Sort project folders cw_config["folders"].sort(key=lambda x: x["path"]) - # Put Odoo folder just before private and top folder + # Put Odoo folder just before private and top folder and map to debugpy odoo = SRC_PATH / "odoo" if odoo.is_dir(): cw_config["folders"].append({"path": str(odoo.relative_to(PROJECT_ROOT))}) @@ -283,10 +390,156 @@ def lint(c, verbose=False): def start(c, detach=True, debugpy=False): """Start environment.""" cmd = "docker-compose up" - if detach: - cmd += " --detach" + with tempfile.NamedTemporaryFile( + mode="w", + suffix=".yaml", + ) as tmp_docker_compose_file: + if debugpy: + # Remove auto-reload + cmd = ( + "docker-compose -f docker-compose.yml " + f"-f {tmp_docker_compose_file.name} up" + ) + _remove_auto_reload( + tmp_docker_compose_file, + orig_file=PROJECT_ROOT / "docker-compose.yml", + ) + if detach: + cmd += " --detach" + with c.cd(str(PROJECT_ROOT)): + result = c.run( + cmd, + pty=True, + env=dict( + UID_ENV, + DOODBA_DEBUGPY_ENABLE=str(int(debugpy)), + ), + ) + if not ("Recreating" in result.stdout or "Starting" in result.stdout): + restart(c) + _logger.info("Waiting for services to spin up...") + time.sleep(SERVICES_WAIT_TIME) + + +@task( + develop, + help={ + "modules": "Comma-separated list of modules to install.", + "core": "Install all core addons. Default: False", + "extra": "Install all extra addons. Default: False", + "private": "Install all private addons. Default: False", + }, +) +def install(c, modules=None, core=False, extra=False, private=False): + """Install Odoo addons + + By default, installs addon from directory being worked on, + unless other options are specified. + """ + if not (modules or core or extra or private): + cur_module = _get_cwd_addon(Path.cwd()) + if not cur_module: + raise exceptions.ParseError( + msg="Odoo addon to install not found. " + "You must provide at least one option for modules" + " or be in a subdirectory of one." + " See --help for details." + ) + modules = cur_module + cmd = "docker-compose run --rm odoo addons init" + if core: + cmd += " --core" + if extra: + cmd += " --extra" + if private: + cmd += " --private" + if modules: + cmd += f" -w {modules}" with c.cd(str(PROJECT_ROOT)): - c.run(cmd, env=dict(UID_ENV, DOODBA_DEBUGPY_ENABLE=str(int(debugpy)))) + c.run( + cmd, + env=UID_ENV, + pty=True, + ) + + +def _test_in_debug_mode(c, odoo_command): + with tempfile.NamedTemporaryFile( + mode="w", suffix=".yaml" + ) as tmp_docker_compose_file: + cmd = ( + "docker-compose -f docker-compose.yml " + f"-f {tmp_docker_compose_file.name} up -d" + ) + _override_docker_command( + "odoo", + odoo_command, + file=tmp_docker_compose_file, + orig_file=Path(str(PROJECT_ROOT), "docker-compose.yml"), + ) + with c.cd(str(PROJECT_ROOT)): + c.run( + cmd, + env=dict( + UID_ENV, + DOODBA_DEBUGPY_ENABLE="1", + ), + ) + _logger.info("Waiting for services to spin up...") + time.sleep(SERVICES_WAIT_TIME) + + +@task( + develop, + help={ + "modules": "Comma-separated list of modules to test.", + "debugpy": "Whether or not to run tests in a VSCode debugging session. " + "Default: False", + "cur-file": "Path to the current file." + " Addon name will be obtained from there to run tests", + "mode": "Mode in which tests run. Options: ['init'(default), 'update']", + }, +) +def test(c, modules=None, debugpy=False, cur_file=None, mode="init"): + """Run Odoo tests + + By default, tests addon from directory being worked on, + unless other options are specified. + + NOTE: Odoo must be restarted manually after this to go back to normal mode + """ + if not modules: + cur_module = _get_cwd_addon(cur_file or Path.cwd()) + if not cur_module: + raise exceptions.ParseError( + msg="Odoo addon to test not found. " + "You must provide at least one option for modules/file " + "or be in a subdirectory of one. " + "See --help for details." + ) + else: + modules = cur_module + odoo_command = ["odoo", "--test-enable", "--stop-after-init", "--workers=0"] + if mode == "init": + odoo_command.append("-i") + elif mode == "update": + odoo_command.append("-u") + else: + raise exceptions.ParseError( + msg="Available modes are 'init' or 'update'. See --help for details." + ) + odoo_command.append(modules) + if debugpy: + _test_in_debug_mode(c, odoo_command) + else: + cmd = ["docker-compose", "run", "--rm", "odoo"] + cmd.extend(odoo_command) + with c.cd(str(PROJECT_ROOT)): + c.run( + " ".join(cmd), + env=UID_ENV, + pty=True, + ) @task( @@ -344,11 +597,22 @@ def restart(c, quick=True): c.run(cmd, env=UID_ENV) -@task(develop) -def logs(c, tail=10): +@task( + develop, + help={ + "container": "Names of the containers from which logs will be obtained." + " You can specify a single one, or several comma-separated names." + " Default: None (show logs for all containers)" + }, +) +def logs(c, tail=10, follow=True, container=None): """Obtain last logs of current environment.""" - cmd = "docker-compose logs -f" + cmd = "docker-compose logs" + if follow: + cmd += " -f" if tail: cmd += f" --tail {tail}" + if container: + cmd += f" {container.replace(',', ' ')}" with c.cd(str(PROJECT_ROOT)): c.run(cmd)