From 51ed8a85a89983823b801c43647732d911cb63b0 Mon Sep 17 00:00:00 2001 From: Glen Whitney Date: Fri, 20 Nov 2020 18:34:06 +0000 Subject: [PATCH] feat: All data from Justworks file read and processed --- .../doctype/jw_import/jw_import.py | 165 ++++++++++++++++-- .../jw_invoice_line/jw_invoice_line.json | 6 +- 2 files changed, 156 insertions(+), 15 deletions(-) diff --git a/justworks_erpnext/justworks_erpnext_connector/doctype/jw_import/jw_import.py b/justworks_erpnext/justworks_erpnext_connector/doctype/jw_import/jw_import.py index 614ec51..9004a01 100644 --- a/justworks_erpnext/justworks_erpnext_connector/doctype/jw_import/jw_import.py +++ b/justworks_erpnext/justworks_erpnext_connector/doctype/jw_import/jw_import.py @@ -5,7 +5,9 @@ from __future__ import unicode_literals import frappe from frappe.model.document import Document +from frappe.utils import get_datetime from frappe.utils.csvutils import read_csv_content +from frappe.utils.data import flt from frappe.utils.dateutils import parse_date from fnmatch import fnmatch as wildcard_match @@ -15,33 +17,66 @@ import re class JWImport(Document): account_mapping_fields = 'charge_type charge_subtype map_to_account'.split() + collection_ref = 'Collection Summary' def process_invoice_file(self): + # See if we should be generating extra collection entries + ccr_flag = self.jw_collection_debits and self.jw_collection_credits fcontent = frappe.get_doc('File', dict(file_url=self.load_from_file)).get_content() rows = read_csv_content(fcontent) header = [frappe.scrub(h) for h in rows.pop(0)] self.set('lines', []) + inv_total = 0.0 + id_idx = header.index('invoice_id') + cur_id = '' + rows.append(['THIS_IS_END_SENTINEL'] * (id_idx+1)) # Load the base data from the file - for row in rows: - rec = self.append('lines', {}) + for row in rows: + # Emit a collection record if invoice is complete + if ccr_flag and cur_id and cur_id != row[id_idx]: + lrec = self.lines[-1] + ctmp = dict(invoice_id=cur_id, + invoice_date=lrec.invoice_date, + debit_date=lrec.debit_date, + reference=self.collection_ref, + amount=inv_total) + crec = self.append('lines', {}) + for k in ctmp: crec.set(k, ctmp[k]) + inv_total = 0.0 + if row[id_idx] == 'THIS_IS_END_SENTINEL': break + + temp = {} for fieldname, val in zip(header, row): if val: if fieldname[-4:] == 'date': - val = parse_date(val) + val = get_datetime(parse_date(val)).date() elif fieldname == 'amount': val = re.sub('^[^.,\d]*', '', val) - rec.set(fieldname, val) + temp[fieldname] = val + if (self.start_invoice_date and temp['invoice_date'] < self.start_invoice_date): + continue + if (self.end_invoice_date and temp['invoice_date'] > self.end_invoice_date): + continue + cur_id = temp['invoice_id'] + rec = self.append('lines', {}) + for k in temp: + rec.set(k, temp[k]) + inv_total += flt(rec.amount) + # Now determine all of the derived data self.map_accounts(self.lines) self.determine_references(self.lines) - #self.check_data(self.lines) - #self.determine_disposition(self.lines) + self.check_data(self.lines) + self.determine_disposition(self.lines) def map_accounts(self, recs): amf = self.account_mapping_fields acc_maps = frappe.get_all('JW Account Mapping', fields=amf) for rec in recs: + if rec.reference == self.collection_ref: + rec.set('account', self.jw_collection_debits) + continue for amap in acc_maps: if all(wildcard_match((rec.get(fld,'') or ''), amap.get(fld)) @@ -82,9 +117,10 @@ class JWImport(Document): # Ok, this is an expense claim: self.search_expense_claim(rec, emp) - # Look for and return an employee matching the reference of the given - # record, if any def search_employee(self, rec): + """Look for and return an employee matching the reference + of the given record, if any + """ fullref = rec.reference (last, comma, restref) = fullref.partition(',') emps = (frappe.get_all('Employee', @@ -100,9 +136,10 @@ class JWImport(Document): closest = max(emps, key=lambda e: len(commonprefix(e, first))) return closest.name - # Look for a purchase invoice with the same supplier and amount - # as rec and fill in the reference_doc if found def search_invoice(self, rec): + """Look for a purchase invoice with the same supplier and + amount as rec and fill in the reference_doc if found + """ # Reference should be a supplier suppliers = (frappe.get_all('Supplier', {'supplier_name': rec.reference}) or @@ -140,9 +177,11 @@ class JWImport(Document): # enough left to be paid and none with matching # amount, so punt. - # Look for an expense claim with given emp and the same amount as - # the given rec, and if one is found, fill in the reference_doc def search_expense_claim(self, rec, emp): + """Look for an expense claim with given emp and the same + amount as the given rec, and if one is found, + fill in the reference_doc + """ claims = frappe.get_all('Expense Claim', filters={'employee':emp, 'company': self.justworks_company}, @@ -170,3 +209,105 @@ class JWImport(Document): return # Hmmm, employee has claims but no large enough unpaid ones # and none with matching amount, so punt. + + def check_data(self, recs): + """Make sure all of the data needed to create entries + is available. + """ + alloc_cache = {} + for rec in recs: + if not rec.account: + rec.data_status = 'No Account' + continue + if rec.reference == self.collection_ref: + rec.data_status = 'OK' + continue + if not rec.reference_doc: + rec.data_status = 'No Party' + continue + if rec.reference_doctype != 'Employee': + rec.data_status = 'OK' + continue + alloc = get_employee_allocation(rec.reference_doc, + rec.invoice_date, alloc_cache) + if sum(a.percentage for a in alloc) != 100: + rec.data_status = 'Bad Alloc' + continue + rec.data_status = 'OK' + + def determine_disposition(self, recs): + for rec in recs: + if rec.data_status != 'OK': + rec.dispostion = 'Data Missing' + continue + + if rec.reference == self.collection_ref: + rec.disposition = 'Collection' + elif rec.reference_doctype == 'Expense Claim': + rec.disposition = 'Pay Expense' + elif rec.reference_doctype == 'Purchase Invoice': + rec.disposition = 'Pay Invoice' + elif self.jw_obligation_account: + rec.disposition = 'Journal' + else: + rec.disposition = 'Ledger Unset' + continue + + if rec.disposition[:3] == 'Pay' and not self.jw_payment_account: + rec.disposition = 'Ledger Unset' + continue + if self.apparent_duplicate(rec): + rec.disposition = 'Duplicate' + + def apparent_duplicate(self, rec): + if rec.disposition == 'Collection': + jes = frappe.get_all('Journal Entry', filters={ + 'company': self.justworks_company, + 'posting_date': rec.debit_date}) + if not jes: return False + je_names = [je.name for je in jes] + return frappe.db.exists({ + 'doctype': 'Journal Entry Account', + 'parent': ('in', je_names), + 'account': self.jw_collection_debits, + 'debit_in_account_currency': rec.amount}) + if rec.disposition[:3] == 'Pay': + pes = frappe.get_all('Payment Entry', filters={ + 'company': self.justworks_company, + 'paid_from': self.jw_payment_account, + 'paid_amount': rec.amount}) + if not pes: return False + pe_names = [pe.name for pe in pes] + return frappe.db.exists({ + 'doctype': 'Payment Entry Reference', + 'parent': ('in', pe_names), + 'reference_name': rec.reference_doc}) + if rec.disposition == 'Journal': + jes = frappe.get_all('Journal Entry', filters={ + 'company': self.justworks_company, + 'posting_date': rec.debit_date}) + if not jes: return False + je_names = [je.name for je in jes] + jeas = frappe.get_all('Journal Entry Account', + filters={ + 'parent': ('in', je_names), + 'account': rec.account + }, + fields=['name','parent']) + jea_parents= [jea.parent for jea in jeas] + return frappe.db.exists({ + 'doctype': 'Journal Entry Account', + 'parent': ('in', jea_parents), + 'account': self.jw_obligation_account}) + # shouldn't get here but just in case + return False + +def get_employee_allocation(emp, date, cache): + if emp in cache: return cache[emp] + alloc = frappe.get_all('JW Employee Allocation', + filters={'employee': emp, 'docstatus': 1}, + fields=['name','percentage','start_date','end_date']) + alloc = [r for r in alloc if (not r.start_date or date >= r.start_date) + and (not r.end_date or date <= r.end_date)] + cache[emp] = alloc + return alloc diff --git a/justworks_erpnext/justworks_erpnext_connector/doctype/jw_invoice_line/jw_invoice_line.json b/justworks_erpnext/justworks_erpnext_connector/doctype/jw_invoice_line/jw_invoice_line.json index f854faf..46baddd 100644 --- a/justworks_erpnext/justworks_erpnext_connector/doctype/jw_invoice_line/jw_invoice_line.json +++ b/justworks_erpnext/justworks_erpnext_connector/doctype/jw_invoice_line/jw_invoice_line.json @@ -111,7 +111,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Data", - "options": "OK\nBad Alloc\nNo Account", + "options": "OK\nNo Account\nNo Party\nBad Alloc", "read_only": 1 }, { @@ -124,7 +124,7 @@ "fieldtype": "Select", "in_list_view": 1, "label": "Disposition", - "options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nDuplicate\nData Missing" + "options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nLedger Unset\nDuplicate\nData Missing" }, { "fieldname": "voucher_section", @@ -186,7 +186,7 @@ "index_web_pages_for_search": 1, "istable": 1, "links": [], - "modified": "2020-11-18 23:21:36.546404", + "modified": "2020-11-19 20:19:32.479219", "modified_by": "Administrator", "module": "Justworks ERPNext Connector", "name": "JW Invoice Line",