You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

264 lines
10 KiB

  1. # -*- coding: utf-8 -*-
  2. """Class to parse camt files."""
  3. # © 2013-2016 Therp BV <http://therp.nl>
  4. # Copyright 2017 Open Net Sàrl
  5. # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
  6. import re
  7. from lxml import etree
  8. from odoo import models
  9. class CamtParser(models.AbstractModel):
  10. _name = 'account.bank.statement.import.camt.parser'
  11. """Parser for camt bank statement import files."""
  12. def parse_amount(self, ns, node):
  13. """Parse element that contains Amount and CreditDebitIndicator."""
  14. if node is None:
  15. return 0.0
  16. sign = 1
  17. amount = 0.0
  18. sign_node = node.xpath('ns:CdtDbtInd', namespaces={'ns': ns})
  19. if sign_node and sign_node[0].text == 'DBIT':
  20. sign = -1
  21. amount_node = node.xpath('ns:Amt', namespaces={'ns': ns})
  22. if amount_node:
  23. amount = sign * float(amount_node[0].text)
  24. return amount
  25. def add_value_from_node(
  26. self, ns, node, xpath_str, obj, attr_name, join_str=None):
  27. """Add value to object from first or all nodes found with xpath.
  28. If xpath_str is a list (or iterable), it will be seen as a series
  29. of search path's in order of preference. The first item that results
  30. in a found node will be used to set a value."""
  31. if not isinstance(xpath_str, (list, tuple)):
  32. xpath_str = [xpath_str]
  33. for search_str in xpath_str:
  34. found_node = node.xpath(search_str, namespaces={'ns': ns})
  35. if found_node:
  36. if join_str is None:
  37. attr_value = found_node[0].text
  38. else:
  39. attr_value = join_str.join([x.text for x in found_node])
  40. obj[attr_name] = attr_value
  41. break
  42. def parse_transaction_details(self, ns, node, transaction):
  43. """Parse TxDtls node."""
  44. # message
  45. self.add_value_from_node(
  46. ns, node, [
  47. './ns:RmtInf/ns:Ustrd',
  48. './ns:AddtlNtryInf',
  49. './ns:Refs/ns:InstrId',
  50. ], transaction, 'note', join_str='\n')
  51. # name
  52. self.add_value_from_node(
  53. ns, node, [
  54. './ns:AddtlTxInf',
  55. ], transaction, 'name', join_str='\n')
  56. # eref
  57. self.add_value_from_node(
  58. ns, node, [
  59. './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref',
  60. './ns:Refs/ns:EndToEndId',
  61. './ns:Ntry/ns:AcctSvcrRef'
  62. ],
  63. transaction, 'ref'
  64. )
  65. amount = self.parse_amount(ns, node)
  66. if amount != 0.0:
  67. transaction['amount'] = amount
  68. # remote party values
  69. party_type = 'Dbtr'
  70. party_type_node = node.xpath(
  71. '../../ns:CdtDbtInd', namespaces={'ns': ns})
  72. if party_type_node and party_type_node[0].text != 'CRDT':
  73. party_type = 'Cdtr'
  74. party_node = node.xpath(
  75. './ns:RltdPties/ns:%s' % party_type, namespaces={'ns': ns})
  76. if party_node:
  77. self.add_value_from_node(
  78. ns, party_node[0], './ns:Nm', transaction, 'partner_name')
  79. self.add_value_from_node(
  80. ns, party_node[0], './ns:PstlAdr/ns:Ctry', transaction,
  81. 'partner_country'
  82. )
  83. address_node = party_node[0].xpath(
  84. './ns:PstlAdr/ns:AdrLine', namespaces={'ns': ns})
  85. if address_node:
  86. transaction['partner_address'] = [address_node[0].text]
  87. # Get remote_account from iban or from domestic account:
  88. account_node = node.xpath(
  89. './ns:RltdPties/ns:%sAcct/ns:Id' % party_type,
  90. namespaces={'ns': ns}
  91. )
  92. if account_node:
  93. iban_node = account_node[0].xpath(
  94. './ns:IBAN', namespaces={'ns': ns})
  95. if iban_node:
  96. transaction['account_number'] = iban_node[0].text
  97. bic_node = node.xpath(
  98. './ns:RltdAgts/ns:%sAgt/ns:FinInstnId/ns:BIC' % party_type,
  99. namespaces={'ns': ns}
  100. )
  101. if bic_node:
  102. transaction['account_bic'] = bic_node[0].text
  103. else:
  104. self.add_value_from_node(
  105. ns, account_node[0], './ns:Othr/ns:Id', transaction,
  106. 'account_number'
  107. )
  108. transaction['data'] = etree.tostring(node)
  109. def parse_entry(self, ns, node):
  110. """Parse an Ntry node and yield transactions"""
  111. transaction = {'name': '/', 'amount': 0} # fallback defaults
  112. self.add_value_from_node(
  113. ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction,
  114. 'transfer_type'
  115. )
  116. self.add_value_from_node(
  117. ns, node, './ns:BookgDt/ns:Dt', transaction, 'date')
  118. self.add_value_from_node(
  119. ns, node, './ns:BookgDt/ns:Dt', transaction, 'execution_date')
  120. self.add_value_from_node(
  121. ns, node, './ns:ValDt/ns:Dt', transaction, 'value_date')
  122. amount = self.parse_amount(ns, node)
  123. if amount != 0.0:
  124. transaction['amount'] = amount
  125. self.add_value_from_node(
  126. ns, node, './ns:AddtlNtryInf', transaction, 'name')
  127. self.add_value_from_node(
  128. ns, node, [
  129. './ns:NtryDtls/ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref',
  130. './ns:NtryDtls/ns:Btch/ns:PmtInfId',
  131. './ns:NtryDtls/ns:TxDtls/ns:Refs/ns:AcctSvcrRef'
  132. ],
  133. transaction, 'ref'
  134. )
  135. details_nodes = node.xpath(
  136. './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns})
  137. if len(details_nodes) == 0:
  138. yield transaction
  139. return
  140. transaction_base = transaction
  141. for node in details_nodes:
  142. transaction = transaction_base.copy()
  143. self.parse_transaction_details(ns, node, transaction)
  144. yield transaction
  145. def get_balance_amounts(self, ns, node):
  146. """Return opening and closing balance.
  147. Depending on kind of balance and statement, the balance might be in a
  148. different kind of node:
  149. OPBD = OpeningBalance
  150. PRCD = PreviousClosingBalance
  151. ITBD = InterimBalance (first ITBD is start-, second is end-balance)
  152. CLBD = ClosingBalance
  153. """
  154. start_balance_node = None
  155. end_balance_node = None
  156. for node_name in ['OPBD', 'PRCD', 'CLBD', 'ITBD']:
  157. code_expr = (
  158. './ns:Bal/ns:Tp/ns:CdOrPrtry/ns:Cd[text()="%s"]/../../..' %
  159. node_name
  160. )
  161. balance_node = node.xpath(code_expr, namespaces={'ns': ns})
  162. if balance_node:
  163. if node_name in ['OPBD', 'PRCD']:
  164. start_balance_node = balance_node[0]
  165. elif node_name == 'CLBD':
  166. end_balance_node = balance_node[0]
  167. else:
  168. if not start_balance_node:
  169. start_balance_node = balance_node[0]
  170. if not end_balance_node:
  171. end_balance_node = balance_node[-1]
  172. return (
  173. self.parse_amount(ns, start_balance_node),
  174. self.parse_amount(ns, end_balance_node)
  175. )
  176. def parse_statement(self, ns, node):
  177. """Parse a single Stmt node."""
  178. result = {}
  179. self.add_value_from_node(
  180. ns, node, [
  181. './ns:Acct/ns:Id/ns:IBAN',
  182. './ns:Acct/ns:Id/ns:Othr/ns:Id',
  183. ], result, 'account_number'
  184. )
  185. self.add_value_from_node(
  186. ns, node, './ns:Id', result, 'name')
  187. self.add_value_from_node(
  188. ns, node, './ns:Dt', result, 'date')
  189. self.add_value_from_node(
  190. ns, node, './ns:Acct/ns:Ccy', result, 'currency')
  191. result['balance_start'], result['balance_end_real'] = (
  192. self.get_balance_amounts(ns, node))
  193. entry_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns})
  194. transactions = []
  195. for entry_node in entry_nodes:
  196. transactions.extend(self.parse_entry(ns, entry_node))
  197. result['transactions'] = transactions
  198. result['date'] = sorted(transactions,
  199. key=lambda x: x['date'],
  200. reverse=True
  201. )[0]['date']
  202. return result
  203. def check_version(self, ns, root):
  204. """Validate validity of camt file."""
  205. # Check wether it is camt at all:
  206. re_camt = re.compile(
  207. r'(^urn:iso:std:iso:20022:tech:xsd:camt.'
  208. r'|^ISO:camt.)'
  209. )
  210. if not re_camt.search(ns):
  211. raise ValueError('no camt: ' + ns)
  212. # Check wether version 052 or 053:
  213. re_camt_version = re.compile(
  214. r'(^urn:iso:std:iso:20022:tech:xsd:camt.053.'
  215. r'|^urn:iso:std:iso:20022:tech:xsd:camt.052.'
  216. r'|^ISO:camt.053.'
  217. r'|^ISO:camt.052.)'
  218. )
  219. if not re_camt_version.search(ns):
  220. raise ValueError('no camt 052 or 053: ' + ns)
  221. # Check GrpHdr element:
  222. root_0_0 = root[0][0].tag[len(ns) + 2:] # strip namespace
  223. if root_0_0 != 'GrpHdr':
  224. raise ValueError('expected GrpHdr, got: ' + root_0_0)
  225. def parse(self, data):
  226. """Parse a camt.052 or camt.053 file."""
  227. try:
  228. root = etree.fromstring(
  229. data, parser=etree.XMLParser(recover=True))
  230. except etree.XMLSyntaxError:
  231. # ABNAmro is known to mix up encodings
  232. root = etree.fromstring(
  233. data.decode('iso-8859-15').encode('utf-8'))
  234. if root is None:
  235. raise ValueError(
  236. 'Not a valid xml file, or not an xml file at all.')
  237. ns = root.tag[1:root.tag.index("}")]
  238. self.check_version(ns, root)
  239. statements = []
  240. currency = None
  241. account_number = None
  242. for node in root[0][1:]:
  243. statement = self.parse_statement(ns, node)
  244. if len(statement['transactions']):
  245. if 'currency' in statement:
  246. currency = statement.pop('currency')
  247. if 'account_number' in statement:
  248. account_number = statement.pop('account_number')
  249. statements.append(statement)
  250. return currency, account_number, statements