From 173b2b2db2eb0a5b4c4f0fc3c9319b1c8d221c41 Mon Sep 17 00:00:00 2001 From: Jairo Llopis Date: Thu, 24 Jan 2019 13:09:37 +0000 Subject: [PATCH] [10.0][IMP] contract: Performance boost :rocket: With this patch we save about 83% of the execution time when generating invoices in batch. # Optimizations made ## Recompute once at the end of the batch This part avoids recomputing many fields per record. Instead, global recomputations are triggered at the end of the block: ```python with _self.env.norecompute(): ... invoices.compute_taxes() _self.recompute() ``` Notice the explicit call to `compute_taxes()`, which was explicit before also, but it was done once per invoice, losing batch-computing boost. ## Disabling prefetch for extra fields It's done in this part: ```python _self = self.with_context(prefetch_fields=False) ``` Prefetching makes sense when we are going to use a lot of fields for a model that has only a few. In our case, we are using not much fields, but the models involved have lots of them. This produces more queries to get those fields, but the queries are noticeably smaller. At the end of the day, it saves a lot of time, which is what matters. ## Disabling track mail creation This part does it: ```diff ctx.update({ + 'mail_notrack': True, 'next_date': next_date, ``` It makes that when creating invoices, we don't create the "Invoice created" message. ## Precomputing price Obtaining `price_unit` from `contract.recurring_invoice_line_ids` was quite expenisve in terms of CPU, and it was being made once per line, each one in a different context, which means also a different cache. Instead of that, lines now share a single context, and are computed before starting the batch. This code precomputes stuff: ```python # Precompute expensive computed fields in batch recurring_lines = _self.mapped("recurring_invoice_line_ids") recurring_lines._fields["price_unit"].determine_value(recurring_lines) ``` And the usage of 2 different environments done inside `_create_invoice()` (`self` and `_self`) guarantee that the invoices are filled with the correct data, but also that the lines use the cached precomputed value instead of having to compute it each time. # Performance gain According to my tests, generating 10 invoices took 62 seconds before, and it takes about 18 seconds now. --- contract/README.rst | 19 ++++++++++ contract/models/account_analytic_account.py | 42 ++++++++++++--------- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/contract/README.rst b/contract/README.rst index 267baaef..cc397553 100644 --- a/contract/README.rst +++ b/contract/README.rst @@ -16,6 +16,25 @@ Configuration To view discount field set *Discount on lines* in user access rights. +You might find that generating all pending invoices at once takes too much +time and produces some performance problems, mostly in cases where you +generate a lot of invoices in little time (i.e. when invoicing thousands +of contracts yearly, and you get to January 1st of the next year). To avoid +this bottleneck, the trick is to **increase the cron frequence and decrease +the contracts batch size**. The counterpart is that some invoices could not +be generated in the exact day you expected. To configure that: + +#. Go to *Settings > Activate the developer mode*. +#. Go to *Settings > Technical > Automation > Scheduled Actions > + Generate Recurring Invoices from Contracts > Edit > Information*. +#. Set a lower interval. For example, change *Interval Unit* to *Hours*. +#. Go to the *Technical Data* tab, and add a batch size in *Arguments*. + For example, it should look like ``(20,)``. +#. Save. + +That's it! From now on, only 20 invoices per hour will be generated. +That should take only a few seconds each hour, and shouln't block other users. + Usage ===== diff --git a/contract/models/account_analytic_account.py b/contract/models/account_analytic_account.py index df26b7b9..3eabbbf5 100644 --- a/contract/models/account_analytic_account.py +++ b/contract/models/account_analytic_account.py @@ -255,12 +255,16 @@ class AccountAnalyticAccount(models.Model): @api.multi def _create_invoice(self): self.ensure_one() - invoice_vals = self._prepare_invoice() - invoice = self.env['account.invoice'].create(invoice_vals) + # Re-read contract with correct company + _self = self.with_context(self.get_invoice_context()) + invoice_vals = _self._prepare_invoice() + invoice = _self.env['account.invoice'].create(invoice_vals) + # Lines are read from an env where expensive values are precomputed for line in self.recurring_invoice_line_ids: - invoice_line_vals = self._prepare_invoice_line(line, invoice.id) - self.env['account.invoice.line'].create(invoice_line_vals) - invoice.compute_taxes() + invoice_line_vals = _self._prepare_invoice_line(line, invoice.id) + _self.env['account.invoice.line'].create(invoice_line_vals) + # Update next invoice date for current contract + _self.recurring_next_date = _self.env.context['next_date'] return invoice @api.multi @@ -287,6 +291,7 @@ class AccountAnalyticAccount(models.Model): relativedelta(days=1)) date_to = date_start ctx.update({ + 'mail_notrack': True, 'next_date': next_date, 'date_format': date_format, 'date_from': date_from, @@ -317,18 +322,21 @@ class AccountAnalyticAccount(models.Model): :return: invoices created """ - invoices = self.env['account.invoice'] - for contract in self: - if limit and len(invoices) >= limit: - break - if not contract.check_dates_valid(): - continue - # Re-read contract with correct company - ctx = contract.get_invoice_context() - invoices |= contract.with_context(ctx)._create_invoice() - contract.write({ - 'recurring_next_date': fields.Date.to_string(ctx['next_date']) - }) + _self = self.with_context(prefetch_fields=False) + invoices = _self.env['account.invoice'] + # Precompute expensive computed fields in batch + recurring_lines = _self.mapped("recurring_invoice_line_ids") + recurring_lines._fields["price_unit"].determine_value(recurring_lines) + # Create invoices + with _self.env.norecompute(): + for contract in _self: + if limit and len(invoices) >= limit: + break + if not contract.check_dates_valid(): + continue + invoices |= contract._create_invoice() + invoices.compute_taxes() + _self.recompute() return invoices @api.model