You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

250 lines
9.6 KiB

8 years ago
  1. # -*- coding: utf-8 -*-
  2. """Class to parse camt files."""
  3. # © 2013-2016 Therp BV <http://therp.nl>
  4. # Copyright 2017 Open Net Sàrl
  5. # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
  6. import re
  7. from lxml import etree
  8. from odoo import models
  9. class CamtParser(models.AbstractModel):
  10. _name = 'account.bank.statement.import.camt.parser'
  11. """Parser for camt bank statement import files."""
  12. def parse_amount(self, ns, node):
  13. """Parse element that contains Amount and CreditDebitIndicator."""
  14. if node is None:
  15. return 0.0
  16. sign = 1
  17. amount = 0.0
  18. sign_node = node.xpath('ns:CdtDbtInd', namespaces={'ns': ns})
  19. if not sign_node:
  20. sign_node = node.xpath(
  21. '../../ns:CdtDbtInd', namespaces={'ns': ns})
  22. if sign_node and sign_node[0].text == 'DBIT':
  23. sign = -1
  24. amount_node = node.xpath('ns:Amt', namespaces={'ns': ns})
  25. if not amount_node:
  26. amount_node = node.xpath(
  27. './ns:AmtDtls/ns:TxAmt/ns:Amt', namespaces={'ns': ns})
  28. if amount_node:
  29. amount = sign * float(amount_node[0].text)
  30. return amount
  31. def add_value_from_node(
  32. self, ns, node, xpath_str, obj, attr_name, join_str=None):
  33. """Add value to object from first or all nodes found with xpath.
  34. If xpath_str is a list (or iterable), it will be seen as a series
  35. of search path's in order of preference. The first item that results
  36. in a found node will be used to set a value."""
  37. if not isinstance(xpath_str, (list, tuple)):
  38. xpath_str = [xpath_str]
  39. for search_str in xpath_str:
  40. found_node = node.xpath(search_str, namespaces={'ns': ns})
  41. if found_node:
  42. if join_str is None:
  43. attr_value = found_node[0].text
  44. else:
  45. attr_value = join_str.join([x.text for x in found_node])
  46. obj[attr_name] = attr_value
  47. break
  48. def parse_transaction_details(self, ns, node, transaction):
  49. """Parse TxDtls node."""
  50. # message
  51. self.add_value_from_node(
  52. ns, node, [
  53. './ns:RmtInf/ns:Ustrd',
  54. './ns:AddtlNtryInf',
  55. './ns:Refs/ns:InstrId',
  56. ], transaction, 'name', join_str='\n')
  57. # name
  58. self.add_value_from_node(
  59. ns, node, [
  60. './ns:AddtlTxInf',
  61. ], transaction, 'note', join_str='\n')
  62. # eref
  63. self.add_value_from_node(
  64. ns, node, [
  65. './ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref',
  66. './ns:Refs/ns:EndToEndId',
  67. './ns:Ntry/ns:AcctSvcrRef'
  68. ],
  69. transaction, 'ref'
  70. )
  71. amount = self.parse_amount(ns, node)
  72. if amount != 0.0:
  73. transaction['amount'] = amount
  74. # remote party values
  75. party_type = 'Dbtr'
  76. party_type_node = node.xpath(
  77. '../../ns:CdtDbtInd', namespaces={'ns': ns})
  78. if party_type_node and party_type_node[0].text != 'CRDT':
  79. party_type = 'Cdtr'
  80. party_node = node.xpath(
  81. './ns:RltdPties/ns:%s' % party_type, namespaces={'ns': ns})
  82. if party_node:
  83. self.add_value_from_node(
  84. ns, party_node[0], './ns:Nm', transaction, 'partner_name')
  85. # Get remote_account from iban or from domestic account:
  86. account_node = node.xpath(
  87. './ns:RltdPties/ns:%sAcct/ns:Id' % party_type,
  88. namespaces={'ns': ns}
  89. )
  90. if account_node:
  91. iban_node = account_node[0].xpath(
  92. './ns:IBAN', namespaces={'ns': ns})
  93. if iban_node:
  94. transaction['account_number'] = iban_node[0].text
  95. else:
  96. self.add_value_from_node(
  97. ns, account_node[0], './ns:Othr/ns:Id', transaction,
  98. 'account_number'
  99. )
  100. def parse_entry(self, ns, node, transaction=None):
  101. """Parse an Ntry node and yield transactions"""
  102. if transaction is None:
  103. transaction = {'name': '/', 'amount': 0} # fallback defaults
  104. self.add_value_from_node(
  105. ns, node, './ns:BookgDt/ns:Dt', transaction, 'date')
  106. amount = self.parse_amount(ns, node)
  107. if amount != 0.0:
  108. transaction['amount'] = amount
  109. self.add_value_from_node(
  110. ns, node, './ns:AddtlNtryInf', transaction, 'name')
  111. self.add_value_from_node(
  112. ns, node, [
  113. './ns:NtryDtls/ns:RmtInf/ns:Strd/ns:CdtrRefInf/ns:Ref',
  114. './ns:NtryDtls/ns:Btch/ns:PmtInfId',
  115. './ns:NtryDtls/ns:TxDtls/ns:Refs/ns:AcctSvcrRef'
  116. ],
  117. transaction, 'ref'
  118. )
  119. details_nodes = node.xpath(
  120. './ns:NtryDtls/ns:TxDtls', namespaces={'ns': ns})
  121. if len(details_nodes) == 0:
  122. yield transaction
  123. return
  124. transaction_base = transaction
  125. for node in details_nodes:
  126. transaction = transaction_base.copy()
  127. self.parse_transaction_details(ns, node, transaction)
  128. yield transaction
  129. def get_balance_amounts(self, ns, node):
  130. """Return opening and closing balance.
  131. Depending on kind of balance and statement, the balance might be in a
  132. different kind of node:
  133. OPBD = OpeningBalance
  134. PRCD = PreviousClosingBalance
  135. ITBD = InterimBalance (first ITBD is start-, second is end-balance)
  136. CLBD = ClosingBalance
  137. """
  138. start_balance_node = None
  139. end_balance_node = None
  140. for node_name in ['OPBD', 'PRCD', 'CLBD', 'ITBD']:
  141. code_expr = (
  142. './ns:Bal/ns:Tp/ns:CdOrPrtry/ns:Cd[text()="%s"]/../../..' %
  143. node_name
  144. )
  145. balance_node = node.xpath(code_expr, namespaces={'ns': ns})
  146. if balance_node:
  147. if node_name in ['OPBD', 'PRCD']:
  148. start_balance_node = balance_node[0]
  149. elif node_name == 'CLBD':
  150. end_balance_node = balance_node[0]
  151. else:
  152. if not start_balance_node:
  153. start_balance_node = balance_node[0]
  154. if not end_balance_node:
  155. end_balance_node = balance_node[-1]
  156. return (
  157. self.parse_amount(ns, start_balance_node),
  158. self.parse_amount(ns, end_balance_node)
  159. )
  160. def parse_statement(self, ns, node):
  161. """Parse a single Stmt node."""
  162. result = {}
  163. self.add_value_from_node(
  164. ns, node, [
  165. './ns:Acct/ns:Id/ns:IBAN',
  166. './ns:Acct/ns:Id/ns:Othr/ns:Id',
  167. ], result, 'account_number'
  168. )
  169. self.add_value_from_node(
  170. ns, node, './ns:Id', result, 'name')
  171. self.add_value_from_node(
  172. ns, node, './ns:Acct/ns:Ccy', result, 'currency')
  173. result['balance_start'], result['balance_end_real'] = (
  174. self.get_balance_amounts(ns, node))
  175. entry_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns})
  176. transactions = []
  177. for entry_node in entry_nodes:
  178. transactions.extend(self.parse_entry(ns, entry_node))
  179. result['transactions'] = transactions
  180. result['date'] = None
  181. if transactions:
  182. result['date'] = sorted(transactions,
  183. key=lambda x: x['date'],
  184. reverse=True
  185. )[0]['date']
  186. return result
  187. def check_version(self, ns, root):
  188. """Validate validity of camt file."""
  189. # Check wether it is camt at all:
  190. re_camt = re.compile(
  191. r'(^urn:iso:std:iso:20022:tech:xsd:camt.'
  192. r'|^ISO:camt.)'
  193. )
  194. if not re_camt.search(ns):
  195. raise ValueError('no camt: ' + ns)
  196. # Check wether version 052 ,053 or 054:
  197. re_camt_version = re.compile(
  198. r'(^urn:iso:std:iso:20022:tech:xsd:camt.054.'
  199. r'|^urn:iso:std:iso:20022:tech:xsd:camt.053.'
  200. r'|^urn:iso:std:iso:20022:tech:xsd:camt.052.'
  201. r'|^ISO:camt.054.'
  202. r'|^ISO:camt.053.'
  203. r'|^ISO:camt.052.)'
  204. )
  205. if not re_camt_version.search(ns):
  206. raise ValueError('no camt 052 or 053 or 054: ' + ns)
  207. # Check GrpHdr element:
  208. root_0_0 = root[0][0].tag[len(ns) + 2:] # strip namespace
  209. if root_0_0 != 'GrpHdr':
  210. raise ValueError('expected GrpHdr, got: ' + root_0_0)
  211. def parse(self, data):
  212. """Parse a camt.052 or camt.053 or camt.054 file."""
  213. try:
  214. root = etree.fromstring(
  215. data, parser=etree.XMLParser(recover=True))
  216. except etree.XMLSyntaxError:
  217. # ABNAmro is known to mix up encodings
  218. root = etree.fromstring(
  219. data.decode('iso-8859-15').encode('utf-8'))
  220. if root is None:
  221. raise ValueError(
  222. 'Not a valid xml file, or not an xml file at all.')
  223. ns = root.tag[1:root.tag.index("}")]
  224. self.check_version(ns, root)
  225. statements = []
  226. currency = None
  227. account_number = None
  228. for node in root[0][1:]:
  229. statement = self.parse_statement(ns, node)
  230. if len(statement['transactions']):
  231. if 'currency' in statement:
  232. currency = statement.pop('currency')
  233. if 'account_number' in statement:
  234. account_number = statement.pop('account_number')
  235. statements.append(statement)
  236. return currency, account_number, statements