diff --git a/.copier-answers.yml b/.copier-answers.yml index 92c3c6e..88ba039 100644 --- a/.copier-answers.yml +++ b/.copier-answers.yml @@ -1,33 +1,38 @@ -# Changes here will be overwritten by Copier -_commit: v2.6.1 +# Changes here will be overwritten by Copier; NEVER EDIT MANUALLY +_commit: v5.0.0 _src_path: gh:Tecnativa/doodba-copier-template backup_deletion: true backup_dst: null backup_email_from: null backup_email_to: null +backup_image_version: latest backup_tz: UTC cidr_whitelist: null domains_prod: null domains_test: null gitlab_url: null -odoo_dbfilter: .* +odoo_dbfilter: ^prod odoo_initial_lang: fr_FR odoo_listdb: true odoo_oci_image: null -odoo_proxy: null +odoo_proxy: other odoo_version: 14.0 paths_without_crawlers: - /web - /website/info +postgres_cidr_whitelist: null postgres_dbname: prod +postgres_exposed: false +postgres_exposed_port: 5432 postgres_username: odoo postgres_version: 10 -project_author: Tecnativa +project_author: Alusage project_license: AGPL-3.0-or-later -project_name: myproject-odoo +project_name: odoo-alusage smtp_canonical_default: null smtp_canonical_domains: null smtp_default_from: null smtp_relay_host: null smtp_relay_port: 587 smtp_relay_user: null +smtp_relay_version: "10" diff --git a/.flake8 b/.flake8 index 44ed868..638a9cd 100644 --- a/.flake8 +++ b/.flake8 @@ -1,5 +1,5 @@ [flake8] -max-line-length = 80 +max-line-length = 88 max-complexity = 16 # B = bugbear # B9 = bugbear opinionated (incl line length) diff --git a/.gitignore b/.gitignore index b178bb5..786a910 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,9 @@ /odoo/custom/src/*/ !/odoo/custom/src/private/ -# Ignore docker-compose.yml by default, to allow easy defaults per clone +# Ignore docker-compose.yml and overrides by default, to allow easy defaults per clone /docker-compose.yml +/docker-compose.override.yml # Compiled formats, cache, temporary files, git garbage **.~ diff --git a/.isort.cfg b/.isort.cfg index 7683bad..0ec187e 100644 --- a/.isort.cfg +++ b/.isort.cfg @@ -10,3 +10,4 @@ known_odoo=odoo known_odoo_addons=odoo.addons sections=FUTURE,STDLIB,THIRDPARTY,ODOO,ODOO_ADDONS,FIRSTPARTY,LOCALFOLDER default_section=THIRDPARTY +ensure_newline_before_comments = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 503de4b..c8beb15 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,8 @@ exclude: | (?x) # Files and folders generated by bots, to avoid loops /static/description/index\.html$| + # Files that fail if changed manually + .*\.(diff|patch)$| # Library files can have extraneous formatting (even minimized) /static/(src/)?lib/| # You don't usually want a bot to modify your legal texts @@ -27,10 +29,22 @@ repos: args: - --addons-dir - odoo/custom/src/private + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: + - --expand-star-imports + - --ignore-init-module-imports + - --in-place + - --remove-all-unused-imports + - --remove-duplicate-keys + - --remove-unused-variables - repo: https://github.com/psf/black rev: 20.8b1 hooks: - id: black + additional_dependencies: ["click<8.1.0"] - repo: https://github.com/asottile/pyupgrade rev: v2.7.2 hooks: @@ -71,20 +85,20 @@ repos: - id: check-xml - id: mixed-line-ending args: ["--fix=lf"] - - repo: https://gitlab.com/pycqa/flake8 + - repo: https://github.com/pycqa/flake8 rev: 3.8.3 hooks: - id: flake8 name: flake8 except __init__.py exclude: /__init__\.py$ - additional_dependencies: ["flake8-bugbear==20.1.4"] + additional_dependencies: ["flake8-bugbear==20.1.4", "importlib-metadata<5.0.0"] - id: flake8 name: flake8 only __init__.py args: ["--extend-ignore=F401"] # ignore unused imports in __init__.py files: /__init__\.py$ - additional_dependencies: ["flake8-bugbear==20.1.4"] + additional_dependencies: ["flake8-bugbear==20.1.4", "importlib-metadata<5.0.0"] - repo: https://github.com/pycqa/pylint - rev: pylint-2.5.3 + rev: v2.11.1 hooks: - id: pylint name: pylint with optional checks @@ -94,16 +108,16 @@ repos: - --exit-zero verbose: true additional_dependencies: - - isort==4.3.4 - - pylint-odoo==3.5.0 + - isort==4.3.21 + - pylint-odoo==5.0.5 - id: pylint name: pylint with mandatory checks args: - --valid_odoo_versions=14.0 - --rcfile=.pylintrc-mandatory additional_dependencies: - - isort==4.3.4 - - pylint-odoo==3.5.0 + - isort==4.3.21 + - pylint-odoo==5.0.5 - repo: https://github.com/pre-commit/mirrors-eslint rev: v7.8.1 hooks: diff --git a/.pylintrc b/.pylintrc index 3f81bf9..7242bac 100644 --- a/.pylintrc +++ b/.pylintrc @@ -4,7 +4,7 @@ score=n [ODOOLINT] readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst" -manifest_required_authors=Tecnativa +manifest_required_authors=Alusage manifest_required_keys=license manifest_deprecated_keys=description,active license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3,OPL-1,OEEL-1 diff --git a/common.yaml b/common.yaml index 82b1fb7..edb0d69 100644 --- a/common.yaml +++ b/common.yaml @@ -12,15 +12,15 @@ services: EMAIL_FROM: "" PGDATABASE: &dbname prod PGUSER: &dbuser "odoo" - DB_FILTER: ".*" - PROXY_MODE: "false" + PROXY_MODE: "true" + LIST_DB: "true" tty: true volumes: - filestore:/var/lib/odoo:z db: image: ghcr.io/tecnativa/postgres-autoconf:10-alpine - shm_size: 512mb + shm_size: 4gb environment: POSTGRES_DB: *dbname POSTGRES_USER: *dbuser @@ -30,4 +30,4 @@ services: - db:/var/lib/postgresql/data:z smtpfake: - image: mailhog/mailhog + image: docker.io/mailhog/mailhog diff --git a/devel.yaml b/devel.yaml index d4ac030..94bc5d6 100644 --- a/devel.yaml +++ b/devel.yaml @@ -2,7 +2,7 @@ version: "2.4" services: odoo_proxy: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest depends_on: - odoo networks: &public @@ -72,7 +72,7 @@ services: POSTGRES_PASSWORD: odoopassword pgweb: - image: sosedoff/pgweb + image: docker.io/sosedoff/pgweb networks: *public ports: - "127.0.0.1:14081:8081" @@ -90,7 +90,7 @@ services: - "127.0.0.1:14025:8025" wdb: - image: kozea/wdb + image: docker.io/kozea/wdb networks: *public ports: - "127.0.0.1:14984:1984" @@ -99,7 +99,7 @@ services: # Whitelist outgoing traffic for tests, reports, etc. proxy_cdnjs_cloudflare_com: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest networks: default: aliases: @@ -110,7 +110,7 @@ services: PRE_RESOLVE: 1 proxy_fonts_googleapis_com: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest networks: default: aliases: @@ -121,7 +121,7 @@ services: PRE_RESOLVE: 1 proxy_fonts_gstatic_com: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest networks: default: aliases: @@ -132,7 +132,7 @@ services: PRE_RESOLVE: 1 proxy_www_google_com: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest networks: default: aliases: @@ -143,7 +143,7 @@ services: PRE_RESOLVE: 1 proxy_www_googleapis_com: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest networks: default: aliases: @@ -154,7 +154,7 @@ services: PRE_RESOLVE: 1 proxy_www_gravatar_com: - image: tecnativa/whitelist + image: ghcr.io/tecnativa/docker-whitelist:latest networks: default: aliases: @@ -166,7 +166,7 @@ services: networks: default: - internal: true + internal: ${DOODBA_NETWORK_INTERNAL-true} public: volumes: diff --git a/odoo/.dockerignore b/odoo/.dockerignore index 7d163b4..46d3bde 100644 --- a/odoo/.dockerignore +++ b/odoo/.dockerignore @@ -2,3 +2,4 @@ custom/src/* !custom/src/private !custom/src/*.* +auto diff --git a/prod.yaml b/prod.yaml index 96b44ca..5a09c83 100644 --- a/prod.yaml +++ b/prod.yaml @@ -10,6 +10,7 @@ services: - .docker/odoo.env - .docker/db-access.env environment: + DB_FILTER: "^prod" DOODBA_ENVIRONMENT: "${DOODBA_ENVIRONMENT-prod}" INITIAL_LANG: "fr_FR" depends_on: diff --git a/setup-devel.yaml b/setup-devel.yaml index 9720547..fa9fabb 100644 --- a/setup-devel.yaml +++ b/setup-devel.yaml @@ -1,6 +1,6 @@ # Use this environment to download all repositories from `repos.yaml` file: # -# export UID="$(id -u $USER)" GID="$(id -g $USER)" UMASK="$(umask)" +# export DOODBA_GITAGGREGATE_UID="$(id -u $USER)" DOODBA_GITAGGREGATE_GID="$(id -g $USER)" DOODBA_UMASK="$(umask)" # docker-compose -f setup-devel.yaml run --rm odoo # # You can clean your git project before if you want to have all really clean: @@ -30,9 +30,9 @@ services: environment: DEPTH_DEFAULT: 100 # XXX Export these variables before running setup to own the files - UID: "${UID:-1000}" - GID: "${GID:-1000}" - UMASK: "$UMASK" + UID: "${DOODBA_GITAGGREGATE_UID:-1000}" + GID: "${DOODBA_GITAGGREGATE_GID:-1000}" + UMASK: "$DOODBA_UMASK" user: root entrypoint: autoaggregate diff --git a/tasks.py b/tasks.py index 4143ecd..bb123cf 100644 --- a/tasks.py +++ b/tasks.py @@ -6,19 +6,39 @@ Contains common helpers to develop using this child project. """ import json import os +import stat import tempfile import time +from datetime import datetime from itertools import chain from logging import getLogger from pathlib import Path from shutil import which from invoke import exceptions, task -from invoke.util import yaml + +try: + import yaml +except ImportError: + from invoke.util import yaml PROJECT_ROOT = Path(__file__).parent.absolute() SRC_PATH = PROJECT_ROOT / "odoo" / "custom" / "src" -UID_ENV = {"GID": str(os.getgid()), "UID": str(os.getuid()), "UMASK": "27"} +UID_ENV = { + "GID": os.environ.get("DOODBA_GID", str(os.getgid())), + "UID": os.environ.get("DOODBA_UID", str(os.getuid())), + "DOODBA_UMASK": os.environ.get("DOODBA_UMASK", "27"), +} +UID_ENV.update( + { + "DOODBA_GITAGGREGATE_GID": os.environ.get( + "DOODBA_GITAGGREGATE_GID", UID_ENV["GID"] + ), + "DOODBA_GITAGGREGATE_UID": os.environ.get( + "DOODBA_GITAGGREGATE_UID", UID_ENV["UID"] + ), + } +) SERVICES_WAIT_TIME = int(os.environ.get("SERVICES_WAIT_TIME", 4)) ODOO_VERSION = float( yaml.safe_load((PROJECT_ROOT / "common.yaml").read_text())["services"]["odoo"][ @@ -59,7 +79,7 @@ def _remove_auto_reload(file, orig_file): def _get_cwd_addon(file): - cwd = Path(file) + cwd = Path(file).resolve() manifest_file = False while PROJECT_ROOT < cwd: manifest_file = (cwd / "__manifest__.py").exists() or ( @@ -108,7 +128,29 @@ def write_code_workspace_file(c, cw_path=None): pass # Nevermind, we start with a new config # Static settings cw_config.setdefault("settings", {}) - cw_config["settings"].update({"search.followSymlinks": False}) + cw_config["settings"].update( + { + "python.autoComplete.extraPaths": [f"{str(SRC_PATH)}/odoo"], + "python.linting.flake8Enabled": True, + "python.linting.ignorePatterns": [f"{str(SRC_PATH)}/odoo/**/*.py"], + "python.linting.pylintArgs": [ + f"--init-hook=\"import sys;sys.path.append('{str(SRC_PATH)}/odoo')\"", + "--load-plugins=pylint_odoo", + ], + "python.linting.pylintEnabled": True, + "python.pythonPath": "python%s" % (2 if ODOO_VERSION < 11 else 3), + "restructuredtext.confPath": "", + "search.followSymlinks": False, + "search.useIgnoreFiles": False, + # Language-specific configurations + "[python]": {"editor.defaultFormatter": "ms-python.python"}, + "[json]": {"editor.defaultFormatter": "esbenp.prettier-vscode"}, + "[jsonc]": {"editor.defaultFormatter": "esbenp.prettier-vscode"}, + "[markdown]": {"editor.defaultFormatter": "esbenp.prettier-vscode"}, + "[yaml]": {"editor.defaultFormatter": "esbenp.prettier-vscode"}, + "[xml]": {"editor.formatOnSave": False}, + } + ) # Launch configurations debugpy_configuration = { "name": "Attach Python debugger to running container", @@ -221,6 +263,24 @@ def write_code_workspace_file(c, cw_path=None): "problemMatcher": [], "options": {"statusbar": {"label": "$(play-circle) Start Odoo"}}, }, + { + "label": "Install current module", + "type": "process", + "command": "invoke", + "args": ["install", "--cur-file", "${file}", "restart"], + "presentation": { + "echo": True, + "reveal": "always", + "focus": True, + "panel": "shared", + "showReuseMessage": True, + "clear": False, + }, + "problemMatcher": [], + "options": { + "statusbar": {"label": "$(symbol-property) Install module"} + }, + }, { "label": "Run Odoo Tests for current module", "type": "process", @@ -306,6 +366,24 @@ def write_code_workspace_file(c, cw_path=None): "problemMatcher": [], "options": {"statusbar": {"label": "$(history) Restart Odoo"}}, }, + { + "label": "See container logs", + "type": "process", + "command": "invoke", + "args": ["logs"], + "presentation": { + "echo": True, + "reveal": "always", + "focus": False, + "panel": "shared", + "showReuseMessage": True, + "clear": False, + }, + "problemMatcher": [], + "options": { + "statusbar": {"label": "$(list-selection) See container logs"} + }, + }, ], } # Sort project folders @@ -329,7 +407,12 @@ def write_code_workspace_file(c, cw_path=None): def develop(c): """Set up a basic development environment.""" # Prepare environment - Path(PROJECT_ROOT, "odoo", "auto", "addons").mkdir(parents=True, exist_ok=True) + auto = Path(PROJECT_ROOT, "odoo", "auto") + addons = auto / "addons" + addons.mkdir(parents=True, exist_ok=True) + # Allow others writing, for podman support + auto.chmod(0o777) + addons.chmod(0o777) with c.cd(str(PROJECT_ROOT)): c.run("git init") c.run("ln -sf devel.yaml docker-compose.yml") @@ -345,7 +428,7 @@ def git_aggregate(c): """ with c.cd(str(PROJECT_ROOT)): c.run( - "docker-compose --file setup-devel.yaml run --rm odoo", + "docker-compose --file setup-devel.yaml run --rm -T odoo", env=UID_ENV, ) write_code_workspace_file(c) @@ -360,23 +443,31 @@ def git_aggregate(c): @task(develop) +def closed_prs(c): + """Test closed PRs from repos.yaml""" + with c.cd(str(PROJECT_ROOT / "odoo/custom/src")): + cmd = "gitaggregate -c {} show-closed-prs".format("repos.yaml") + c.run(cmd, env=UID_ENV, pty=True) + + +@task() def img_build(c, pull=True): """Build docker images.""" cmd = "docker-compose build" if pull: cmd += " --pull" with c.cd(str(PROJECT_ROOT)): - c.run(cmd, env=UID_ENV) + c.run(cmd, env=UID_ENV, pty=True) -@task(develop) +@task() def img_pull(c): """Pull docker images.""" with c.cd(str(PROJECT_ROOT)): - c.run("docker-compose pull") + c.run("docker-compose pull", pty=True) -@task(develop) +@task() def lint(c, verbose=False): """Lint & format source code.""" cmd = "pre-commit run --show-diff-on-failure --all-files --color=always" @@ -386,7 +477,7 @@ def lint(c, verbose=False): c.run(cmd) -@task(develop) +@task() def start(c, detach=True, debugpy=False): """Start environment.""" cmd = "docker-compose up" @@ -415,29 +506,43 @@ def start(c, detach=True, debugpy=False): DOODBA_DEBUGPY_ENABLE=str(int(debugpy)), ), ) - if not ("Recreating" in result.stdout or "Starting" in result.stdout): + if not ( + "Recreating" in result.stdout + or "Starting" in result.stdout + or "Creating" in result.stdout + ): restart(c) _logger.info("Waiting for services to spin up...") time.sleep(SERVICES_WAIT_TIME) @task( - develop, help={ "modules": "Comma-separated list of modules to install.", "core": "Install all core addons. Default: False", "extra": "Install all extra addons. Default: False", "private": "Install all private addons. Default: False", + "enterprise": "Install all enterprise addons. Default: False", + "cur-file": "Path to the current file." + " Addon name will be obtained from there to install.", }, ) -def install(c, modules=None, core=False, extra=False, private=False): +def install( + c, + modules=None, + cur_file=None, + core=False, + extra=False, + private=False, + enterprise=False, +): """Install Odoo addons By default, installs addon from directory being worked on, unless other options are specified. """ - if not (modules or core or extra or private): - cur_module = _get_cwd_addon(Path.cwd()) + if not (modules or core or extra or private or enterprise): + cur_module = _get_cwd_addon(cur_file or Path.cwd()) if not cur_module: raise exceptions.ParseError( msg="Odoo addon to install not found. " @@ -453,9 +558,12 @@ def install(c, modules=None, core=False, extra=False, private=False): cmd += " --extra" if private: cmd += " --private" + if enterprise: + cmd += " --enterprise" if modules: cmd += f" -w {modules}" with c.cd(str(PROJECT_ROOT)): + c.run("docker-compose stop odoo") c.run( cmd, env=UID_ENV, @@ -463,6 +571,71 @@ def install(c, modules=None, core=False, extra=False, private=False): ) +@task( + help={ + "modules": "Comma-separated list of modules to uninstall.", + }, +) +def uninstall( + c, + modules=None, + cur_file=None, +): + """Uninstall Odoo addons + + By default, uninstalls addon from directory being worked on, + unless other options are specified. + """ + if not modules: + cur_module = _get_cwd_addon(cur_file or Path.cwd()) + if not cur_module: + raise exceptions.ParseError( + msg="Odoo addon to uninstall not found. " + "You must provide at least one option for modules" + " or be in a subdirectory of one." + " See --help for details." + ) + modules = cur_module + cmd = ( + f"docker-compose run --rm odoo click-odoo-uninstall -m {modules or cur_module}" + ) + with c.cd(str(PROJECT_ROOT)): + c.run( + cmd, + env=UID_ENV, + pty=True, + ) + + +def _get_module_dependencies( + c, modules=None, core=False, extra=False, private=False, enterprise=False +): + """Returns a list of the addons' dependencies + + By default, refers to the addon from directory being worked on, + unless other options are specified. + """ + # Get list of dependencies for addon + cmd = "docker-compose run --rm odoo addons list --dependencies" + if core: + cmd += " --core" + if extra: + cmd += " --extra" + if private: + cmd += " --private" + if enterprise: + cmd += " --enterprise" + if modules: + cmd += f" -w {modules}" + with c.cd(str(PROJECT_ROOT)): + dependencies = c.run( + cmd, + env=UID_ENV, + hide="stdout", + ).stdout.splitlines()[-1] + return dependencies + + def _test_in_debug_mode(c, odoo_command): with tempfile.NamedTemporaryFile( mode="w", suffix=".yaml" @@ -484,23 +657,80 @@ def _test_in_debug_mode(c, odoo_command): UID_ENV, DOODBA_DEBUGPY_ENABLE="1", ), + pty=True, ) _logger.info("Waiting for services to spin up...") time.sleep(SERVICES_WAIT_TIME) +def _get_module_list( + c, + modules=None, + core=False, + extra=False, + private=False, + enterprise=False, + only_installable=True, +): + """Returns a list of addons according to the passed parameters. + + By default, refers to the addon from directory being worked on, + unless other options are specified. + """ + # Get list of dependencies for addon + cmd = "docker-compose run --rm odoo addons list" + if core: + cmd += " --core" + if extra: + cmd += " --extra" + if private: + cmd += " --private" + if enterprise: + cmd += " --enterprise" + if modules: + cmd += f" -w {modules}" + if only_installable: + cmd += " --installable" + with c.cd(str(PROJECT_ROOT)): + module_list = c.run( + cmd, + env=UID_ENV, + pty=True, + hide="stdout", + ).stdout.splitlines()[-1] + return module_list + + @task( - develop, help={ "modules": "Comma-separated list of modules to test.", + "core": "Test all core addons. Default: False", + "extra": "Test all extra addons. Default: False", + "private": "Test all private addons. Default: False", + "enterprise": "Test all enterprise addons. Default: False", + "skip": "List of addons to skip. Default: []", "debugpy": "Whether or not to run tests in a VSCode debugging session. " "Default: False", "cur-file": "Path to the current file." " Addon name will be obtained from there to run tests", "mode": "Mode in which tests run. Options: ['init'(default), 'update']", + "db_filter": "DB_FILTER regex to pass to the test container Set to ''" + " to disable. Default: '^devel$'", }, ) -def test(c, modules=None, debugpy=False, cur_file=None, mode="init"): +def test( + c, + modules=None, + core=False, + extra=False, + private=False, + enterprise=False, + skip="", + debugpy=False, + cur_file=None, + mode="init", + db_filter="^devel$", +): """Run Odoo tests By default, tests addon from directory being worked on, @@ -508,17 +738,18 @@ def test(c, modules=None, debugpy=False, cur_file=None, mode="init"): NOTE: Odoo must be restarted manually after this to go back to normal mode """ - if not modules: + if not (modules or core or extra or private or enterprise): cur_module = _get_cwd_addon(cur_file or Path.cwd()) if not cur_module: raise exceptions.ParseError( - msg="Odoo addon to test not found. " - "You must provide at least one option for modules/file " - "or be in a subdirectory of one. " - "See --help for details." + msg="Odoo addon to install not found. " + "You must provide at least one option for modules" + " or be in a subdirectory of one." + " See --help for details." ) - else: - modules = cur_module + modules = cur_module + else: + modules = _get_module_list(c, modules, core, extra, private, enterprise) odoo_command = ["odoo", "--test-enable", "--stop-after-init", "--workers=0"] if mode == "init": odoo_command.append("-i") @@ -528,11 +759,30 @@ def test(c, modules=None, debugpy=False, cur_file=None, mode="init"): raise exceptions.ParseError( msg="Available modes are 'init' or 'update'. See --help for details." ) + # Skip test in some modules + modules_list = modules.split(",") + for m_to_skip in skip.split(","): + if not m_to_skip: + continue + if m_to_skip not in modules: + _logger.warn( + "%s not found in the list of addons to test: %s" % (m_to_skip, modules) + ) + modules_list.remove(m_to_skip) + modules = ",".join(modules_list) odoo_command.append(modules) + if ODOO_VERSION >= 12: + # Limit tests to explicit list + # Filter spec format (comma-separated) + # [-][tag][/module][:class][.method] + odoo_command.extend(["--test-tags", "/" + ",/".join(modules_list)]) if debugpy: _test_in_debug_mode(c, odoo_command) else: - cmd = ["docker-compose", "run", "--rm", "odoo"] + cmd = ["docker-compose", "run", "--rm"] + if db_filter: + cmd.extend(["-e", "DB_FILTER='%s'" % db_filter]) + cmd.append("odoo") cmd.extend(odoo_command) with c.cd(str(PROJECT_ROOT)): c.run( @@ -543,33 +793,53 @@ def test(c, modules=None, debugpy=False, cur_file=None, mode="init"): @task( - develop, help={"purge": "Remove all related containers, networks images and volumes"}, ) def stop(c, purge=False): """Stop and (optionally) purge environment.""" - cmd = "docker-compose" + cmd = "docker-compose down --remove-orphans" if purge: - cmd += " down --remove-orphans --rmi local --volumes" - else: - cmd += " stop" + cmd += " --rmi local --volumes" with c.cd(str(PROJECT_ROOT)): - c.run(cmd) + c.run(cmd, pty=True) @task( - develop, help={ "dbname": "The DB that will be DESTROYED and recreated. Default: 'devel'.", "modules": "Comma-separated list of modules to install. Default: 'base'.", + "core": "Install all core addons. Default: False", + "extra": "Install all extra addons. Default: False", + "private": "Install all private addons. Default: False", + "enterprise": "Install all enterprise addons. Default: False", + "populate": "Run preparedb task right after (only available for v11+)." + " Default: True", + "dependencies": "Install only the dependencies of the specified addons." + "Default: False", }, ) -def resetdb(c, modules="base", dbname="devel"): +def resetdb( + c, + modules=None, + core=False, + extra=False, + private=False, + enterprise=False, + dbname="devel", + populate=True, + dependencies=False, +): """Reset the specified database with the specified modules. Uses click-odoo-initdb behind the scenes, which has a caching system that makes DB resets quicker. See its docs for more info. """ + if dependencies: + modules = _get_module_dependencies(c, modules, core, extra, private, enterprise) + elif core or extra or private or enterprise: + modules = _get_module_list(c, modules, core, extra, private, enterprise) + else: + modules = modules or "base" with c.cd(str(PROJECT_ROOT)): c.run("docker-compose stop odoo", pty=True) _run = "docker-compose run --rm -l traefik.enable=false odoo" @@ -584,9 +854,34 @@ def resetdb(c, modules="base", dbname="devel"): env=UID_ENV, pty=True, ) + if populate and ODOO_VERSION < 11: + _logger.warn( + "Skipping populate task as it is not available in v%s" % ODOO_VERSION + ) + populate = False + if populate: + preparedb(c) -@task(develop) +@task() +def preparedb(c): + """Run the `preparedb` script inside the container + + Populates the DB with some helpful config + """ + if ODOO_VERSION < 11: + raise exceptions.PlatformError( + "The preparedb script is not available for Doodba environments bellow v11." + ) + with c.cd(str(PROJECT_ROOT)): + c.run( + "docker-compose run --rm -l traefik.enable=false odoo preparedb", + env=UID_ENV, + pty=True, + ) + + +@task() def restart(c, quick=True): """Restart odoo container(s).""" cmd = "docker-compose restart" @@ -594,11 +889,10 @@ def restart(c, quick=True): cmd = f"{cmd} -t0" cmd = f"{cmd} odoo odoo_proxy" with c.cd(str(PROJECT_ROOT)): - c.run(cmd, env=UID_ENV) + c.run(cmd, env=UID_ENV, pty=True) @task( - develop, help={ "container": "Names of the containers from which logs will be obtained." " You can specify a single one, or several comma-separated names." @@ -615,4 +909,132 @@ def logs(c, tail=10, follow=True, container=None): if container: cmd += f" {container.replace(',', ' ')}" with c.cd(str(PROJECT_ROOT)): - c.run(cmd) + c.run(cmd, pty=True) + + +@task +def after_update(c): + """Execute some actions after a copier update or init""" + # Make custom build scripts executable + if ODOO_VERSION < 11: + files = ( + Path(PROJECT_ROOT, "odoo", "custom", "build.d", "20-update-pg-repos"), + Path(PROJECT_ROOT, "odoo", "custom", "build.d", "10-fix-certs"), + ) + for script_file in files: + # Ignore if, for some reason, the file didn't end up in the generated + # project despite of the correct version (e.g. Copier exclusions) + if not script_file.exists(): + continue + cur_stat = script_file.stat() + # Like chmod ug+x + script_file.chmod(cur_stat.st_mode | stat.S_IXUSR | stat.S_IXGRP) + else: + # Remove version-specific build scripts if the copier update didn't + # HACK: https://github.com/copier-org/copier/issues/461 + files = ( + Path(PROJECT_ROOT, "odoo", "custom", "build.d", "20-update-pg-repos"), + Path(PROJECT_ROOT, "odoo", "custom", "build.d", "10-fix-certs"), + ) + for script_file in files: + # missing_ok argument would take care of this, but it was only added for + # Python 3.8 + if script_file.exists(): + script_file.unlink() + + +@task( + help={ + "source_db": "The source DB name. Default: 'devel'.", + "destination_db": "The destination DB name. Default: '[SOURCE_DB_NAME]-[CURRENT_DATE]'", + }, +) +def snapshot( + c, + source_db="devel", + destination_db=None, +): + """Snapshot current database and filestore. + + Uses click-odoo-copydb behind the scenes to make a snapshot. + """ + if not destination_db: + destination_db = "%s-%s" % ( + source_db, + datetime.now().strftime("%Y_%m_%d-%H_%M"), + ) + with c.cd(str(PROJECT_ROOT)): + cur_state = c.run("docker-compose stop odoo db", pty=True).stdout + _logger.info("Snapshoting current %s DB to %s" % (source_db, destination_db)) + _run = "docker-compose run --rm -l traefik.enable=false odoo" + c.run( + f"{_run} click-odoo-copydb {source_db} {destination_db}", + env=UID_ENV, + pty=True, + ) + if "Stopping" in cur_state: + # Restart services if they were previously active + c.run("docker-compose start odoo db", pty=True) + + +@task( + help={ + "snapshot_name": "The snapshot name. If not provided," + "the script will try to find the last snapshot" + " that starts with the destination_db name", + "destination_db": "The destination DB name. Default: 'devel'", + }, +) +def restore_snapshot( + c, + snapshot_name=None, + destination_db="devel", +): + """Restore database and filestore snapshot. + + Uses click-odoo-copydb behind the scenes to restore a DB snapshot. + """ + with c.cd(str(PROJECT_ROOT)): + cur_state = c.run("docker-compose stop odoo db", pty=True).stdout + if not snapshot_name: + # List DBs + res = c.run( + "docker-compose run --rm -e LOG_LEVEL=WARNING odoo psql -tc" + " 'SELECT datname FROM pg_database;'", + env=UID_ENV, + hide="stdout", + ) + db_list = [] + for db in res.stdout.splitlines(): + # Parse and filter DB List + if not db.lstrip().startswith(destination_db): + continue + db_name = db.lstrip() + try: + db_date = datetime.strptime( + db_name.lstrip(destination_db + "-"), "%Y_%m_%d-%H_%M" + ) + db_list.append((db_name, db_date)) + except ValueError: + continue + snapshot_name = max(db_list, key=lambda x: x[1])[0] + if not snapshot_name: + raise exceptions.PlatformError( + "No snapshot found for destination_db %s" % destination_db + ) + _logger.info("Restoring snapshot %s to %s" % (snapshot_name, destination_db)) + _run = "docker-compose run --rm -l traefik.enable=false odoo" + c.run( + f"{_run} click-odoo-dropdb {destination_db}", + env=UID_ENV, + warn=True, + pty=True, + ) + c.run( + f"{_run} click-odoo-copydb {snapshot_name} {destination_db}", + env=UID_ENV, + pty=True, + ) + if "Stopping" in cur_state: + # Restart services if they were previously active + c.run("docker-compose start odoo db", pty=True) diff --git a/test.yaml b/test.yaml index 862299e..7fda66e 100644 --- a/test.yaml +++ b/test.yaml @@ -55,7 +55,7 @@ services: networks: default: - internal: true + internal: ${DOODBA_NETWORK_INTERNAL-true} driver_opts: encrypted: 1