feat: All data from Justworks file read and processed

This commit is contained in:
Glen Whitney 2020-11-20 18:34:06 +00:00
parent b9106e89d4
commit 51ed8a85a8
2 changed files with 156 additions and 15 deletions

View File

@ -5,7 +5,9 @@
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.utils import get_datetime
from frappe.utils.csvutils import read_csv_content
from frappe.utils.data import flt
from frappe.utils.dateutils import parse_date
from fnmatch import fnmatch as wildcard_match
@ -15,33 +17,66 @@ import re
class JWImport(Document):
account_mapping_fields = 'charge_type charge_subtype map_to_account'.split()
collection_ref = 'Collection Summary'
def process_invoice_file(self):
# See if we should be generating extra collection entries
ccr_flag = self.jw_collection_debits and self.jw_collection_credits
fcontent = frappe.get_doc('File',
dict(file_url=self.load_from_file)).get_content()
rows = read_csv_content(fcontent)
header = [frappe.scrub(h) for h in rows.pop(0)]
self.set('lines', [])
inv_total = 0.0
id_idx = header.index('invoice_id')
cur_id = ''
rows.append(['THIS_IS_END_SENTINEL'] * (id_idx+1))
# Load the base data from the file
for row in rows:
rec = self.append('lines', {})
for row in rows:
# Emit a collection record if invoice is complete
if ccr_flag and cur_id and cur_id != row[id_idx]:
lrec = self.lines[-1]
ctmp = dict(invoice_id=cur_id,
invoice_date=lrec.invoice_date,
debit_date=lrec.debit_date,
reference=self.collection_ref,
amount=inv_total)
crec = self.append('lines', {})
for k in ctmp: crec.set(k, ctmp[k])
inv_total = 0.0
if row[id_idx] == 'THIS_IS_END_SENTINEL': break
temp = {}
for fieldname, val in zip(header, row):
if val:
if fieldname[-4:] == 'date':
val = parse_date(val)
val = get_datetime(parse_date(val)).date()
elif fieldname == 'amount':
val = re.sub('^[^.,\d]*', '', val)
rec.set(fieldname, val)
temp[fieldname] = val
if (self.start_invoice_date and temp['invoice_date'] < self.start_invoice_date):
continue
if (self.end_invoice_date and temp['invoice_date'] > self.end_invoice_date):
continue
cur_id = temp['invoice_id']
rec = self.append('lines', {})
for k in temp:
rec.set(k, temp[k])
inv_total += flt(rec.amount)
# Now determine all of the derived data
self.map_accounts(self.lines)
self.determine_references(self.lines)
#self.check_data(self.lines)
#self.determine_disposition(self.lines)
self.check_data(self.lines)
self.determine_disposition(self.lines)
def map_accounts(self, recs):
amf = self.account_mapping_fields
acc_maps = frappe.get_all('JW Account Mapping', fields=amf)
for rec in recs:
if rec.reference == self.collection_ref:
rec.set('account', self.jw_collection_debits)
continue
for amap in acc_maps:
if all(wildcard_match((rec.get(fld,'') or ''),
amap.get(fld))
@ -82,9 +117,10 @@ class JWImport(Document):
# Ok, this is an expense claim:
self.search_expense_claim(rec, emp)
# Look for and return an employee matching the reference of the given
# record, if any
def search_employee(self, rec):
"""Look for and return an employee matching the reference
of the given record, if any
"""
fullref = rec.reference
(last, comma, restref) = fullref.partition(',')
emps = (frappe.get_all('Employee',
@ -100,9 +136,10 @@ class JWImport(Document):
closest = max(emps, key=lambda e: len(commonprefix(e, first)))
return closest.name
# Look for a purchase invoice with the same supplier and amount
# as rec and fill in the reference_doc if found
def search_invoice(self, rec):
"""Look for a purchase invoice with the same supplier and
amount as rec and fill in the reference_doc if found
"""
# Reference should be a supplier
suppliers = (frappe.get_all('Supplier',
{'supplier_name': rec.reference}) or
@ -140,9 +177,11 @@ class JWImport(Document):
# enough left to be paid and none with matching
# amount, so punt.
# Look for an expense claim with given emp and the same amount as
# the given rec, and if one is found, fill in the reference_doc
def search_expense_claim(self, rec, emp):
"""Look for an expense claim with given emp and the same
amount as the given rec, and if one is found,
fill in the reference_doc
"""
claims = frappe.get_all('Expense Claim',
filters={'employee':emp,
'company': self.justworks_company},
@ -170,3 +209,105 @@ class JWImport(Document):
return
# Hmmm, employee has claims but no large enough unpaid ones
# and none with matching amount, so punt.
def check_data(self, recs):
"""Make sure all of the data needed to create entries
is available.
"""
alloc_cache = {}
for rec in recs:
if not rec.account:
rec.data_status = 'No Account'
continue
if rec.reference == self.collection_ref:
rec.data_status = 'OK'
continue
if not rec.reference_doc:
rec.data_status = 'No Party'
continue
if rec.reference_doctype != 'Employee':
rec.data_status = 'OK'
continue
alloc = get_employee_allocation(rec.reference_doc,
rec.invoice_date, alloc_cache)
if sum(a.percentage for a in alloc) != 100:
rec.data_status = 'Bad Alloc'
continue
rec.data_status = 'OK'
def determine_disposition(self, recs):
for rec in recs:
if rec.data_status != 'OK':
rec.dispostion = 'Data Missing'
continue
if rec.reference == self.collection_ref:
rec.disposition = 'Collection'
elif rec.reference_doctype == 'Expense Claim':
rec.disposition = 'Pay Expense'
elif rec.reference_doctype == 'Purchase Invoice':
rec.disposition = 'Pay Invoice'
elif self.jw_obligation_account:
rec.disposition = 'Journal'
else:
rec.disposition = 'Ledger Unset'
continue
if rec.disposition[:3] == 'Pay' and not self.jw_payment_account:
rec.disposition = 'Ledger Unset'
continue
if self.apparent_duplicate(rec):
rec.disposition = 'Duplicate'
def apparent_duplicate(self, rec):
if rec.disposition == 'Collection':
jes = frappe.get_all('Journal Entry', filters={
'company': self.justworks_company,
'posting_date': rec.debit_date})
if not jes: return False
je_names = [je.name for je in jes]
return frappe.db.exists({
'doctype': 'Journal Entry Account',
'parent': ('in', je_names),
'account': self.jw_collection_debits,
'debit_in_account_currency': rec.amount})
if rec.disposition[:3] == 'Pay':
pes = frappe.get_all('Payment Entry', filters={
'company': self.justworks_company,
'paid_from': self.jw_payment_account,
'paid_amount': rec.amount})
if not pes: return False
pe_names = [pe.name for pe in pes]
return frappe.db.exists({
'doctype': 'Payment Entry Reference',
'parent': ('in', pe_names),
'reference_name': rec.reference_doc})
if rec.disposition == 'Journal':
jes = frappe.get_all('Journal Entry', filters={
'company': self.justworks_company,
'posting_date': rec.debit_date})
if not jes: return False
je_names = [je.name for je in jes]
jeas = frappe.get_all('Journal Entry Account',
filters={
'parent': ('in', je_names),
'account': rec.account
},
fields=['name','parent'])
jea_parents= [jea.parent for jea in jeas]
return frappe.db.exists({
'doctype': 'Journal Entry Account',
'parent': ('in', jea_parents),
'account': self.jw_obligation_account})
# shouldn't get here but just in case
return False
def get_employee_allocation(emp, date, cache):
if emp in cache: return cache[emp]
alloc = frappe.get_all('JW Employee Allocation',
filters={'employee': emp, 'docstatus': 1},
fields=['name','percentage','start_date','end_date'])
alloc = [r for r in alloc if (not r.start_date or date >= r.start_date)
and (not r.end_date or date <= r.end_date)]
cache[emp] = alloc
return alloc

View File

@ -111,7 +111,7 @@
"fieldtype": "Select",
"in_list_view": 1,
"label": "Data",
"options": "OK\nBad Alloc\nNo Account",
"options": "OK\nNo Account\nNo Party\nBad Alloc",
"read_only": 1
},
{
@ -124,7 +124,7 @@
"fieldtype": "Select",
"in_list_view": 1,
"label": "Disposition",
"options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nDuplicate\nData Missing"
"options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nLedger Unset\nDuplicate\nData Missing"
},
{
"fieldname": "voucher_section",
@ -186,7 +186,7 @@
"index_web_pages_for_search": 1,
"istable": 1,
"links": [],
"modified": "2020-11-18 23:21:36.546404",
"modified": "2020-11-19 20:19:32.479219",
"modified_by": "Administrator",
"module": "Justworks ERPNext Connector",
"name": "JW Invoice Line",