feat: All data from Justworks file read and processed
This commit is contained in:
parent
b9106e89d4
commit
51ed8a85a8
@ -5,7 +5,9 @@
|
||||
from __future__ import unicode_literals
|
||||
import frappe
|
||||
from frappe.model.document import Document
|
||||
from frappe.utils import get_datetime
|
||||
from frappe.utils.csvutils import read_csv_content
|
||||
from frappe.utils.data import flt
|
||||
from frappe.utils.dateutils import parse_date
|
||||
|
||||
from fnmatch import fnmatch as wildcard_match
|
||||
@ -15,33 +17,66 @@ import re
|
||||
|
||||
class JWImport(Document):
|
||||
account_mapping_fields = 'charge_type charge_subtype map_to_account'.split()
|
||||
collection_ref = 'Collection Summary'
|
||||
|
||||
def process_invoice_file(self):
|
||||
# See if we should be generating extra collection entries
|
||||
ccr_flag = self.jw_collection_debits and self.jw_collection_credits
|
||||
fcontent = frappe.get_doc('File',
|
||||
dict(file_url=self.load_from_file)).get_content()
|
||||
rows = read_csv_content(fcontent)
|
||||
header = [frappe.scrub(h) for h in rows.pop(0)]
|
||||
self.set('lines', [])
|
||||
inv_total = 0.0
|
||||
id_idx = header.index('invoice_id')
|
||||
cur_id = ''
|
||||
rows.append(['THIS_IS_END_SENTINEL'] * (id_idx+1))
|
||||
# Load the base data from the file
|
||||
for row in rows:
|
||||
rec = self.append('lines', {})
|
||||
for row in rows:
|
||||
# Emit a collection record if invoice is complete
|
||||
if ccr_flag and cur_id and cur_id != row[id_idx]:
|
||||
lrec = self.lines[-1]
|
||||
ctmp = dict(invoice_id=cur_id,
|
||||
invoice_date=lrec.invoice_date,
|
||||
debit_date=lrec.debit_date,
|
||||
reference=self.collection_ref,
|
||||
amount=inv_total)
|
||||
crec = self.append('lines', {})
|
||||
for k in ctmp: crec.set(k, ctmp[k])
|
||||
inv_total = 0.0
|
||||
if row[id_idx] == 'THIS_IS_END_SENTINEL': break
|
||||
|
||||
temp = {}
|
||||
for fieldname, val in zip(header, row):
|
||||
if val:
|
||||
if fieldname[-4:] == 'date':
|
||||
val = parse_date(val)
|
||||
val = get_datetime(parse_date(val)).date()
|
||||
elif fieldname == 'amount':
|
||||
val = re.sub('^[^.,\d]*', '', val)
|
||||
rec.set(fieldname, val)
|
||||
temp[fieldname] = val
|
||||
if (self.start_invoice_date and temp['invoice_date'] < self.start_invoice_date):
|
||||
continue
|
||||
if (self.end_invoice_date and temp['invoice_date'] > self.end_invoice_date):
|
||||
continue
|
||||
cur_id = temp['invoice_id']
|
||||
rec = self.append('lines', {})
|
||||
for k in temp:
|
||||
rec.set(k, temp[k])
|
||||
inv_total += flt(rec.amount)
|
||||
|
||||
# Now determine all of the derived data
|
||||
self.map_accounts(self.lines)
|
||||
self.determine_references(self.lines)
|
||||
#self.check_data(self.lines)
|
||||
#self.determine_disposition(self.lines)
|
||||
self.check_data(self.lines)
|
||||
self.determine_disposition(self.lines)
|
||||
|
||||
def map_accounts(self, recs):
|
||||
amf = self.account_mapping_fields
|
||||
acc_maps = frappe.get_all('JW Account Mapping', fields=amf)
|
||||
for rec in recs:
|
||||
if rec.reference == self.collection_ref:
|
||||
rec.set('account', self.jw_collection_debits)
|
||||
continue
|
||||
for amap in acc_maps:
|
||||
if all(wildcard_match((rec.get(fld,'') or ''),
|
||||
amap.get(fld))
|
||||
@ -82,9 +117,10 @@ class JWImport(Document):
|
||||
# Ok, this is an expense claim:
|
||||
self.search_expense_claim(rec, emp)
|
||||
|
||||
# Look for and return an employee matching the reference of the given
|
||||
# record, if any
|
||||
def search_employee(self, rec):
|
||||
"""Look for and return an employee matching the reference
|
||||
of the given record, if any
|
||||
"""
|
||||
fullref = rec.reference
|
||||
(last, comma, restref) = fullref.partition(',')
|
||||
emps = (frappe.get_all('Employee',
|
||||
@ -100,9 +136,10 @@ class JWImport(Document):
|
||||
closest = max(emps, key=lambda e: len(commonprefix(e, first)))
|
||||
return closest.name
|
||||
|
||||
# Look for a purchase invoice with the same supplier and amount
|
||||
# as rec and fill in the reference_doc if found
|
||||
def search_invoice(self, rec):
|
||||
"""Look for a purchase invoice with the same supplier and
|
||||
amount as rec and fill in the reference_doc if found
|
||||
"""
|
||||
# Reference should be a supplier
|
||||
suppliers = (frappe.get_all('Supplier',
|
||||
{'supplier_name': rec.reference}) or
|
||||
@ -140,9 +177,11 @@ class JWImport(Document):
|
||||
# enough left to be paid and none with matching
|
||||
# amount, so punt.
|
||||
|
||||
# Look for an expense claim with given emp and the same amount as
|
||||
# the given rec, and if one is found, fill in the reference_doc
|
||||
def search_expense_claim(self, rec, emp):
|
||||
"""Look for an expense claim with given emp and the same
|
||||
amount as the given rec, and if one is found,
|
||||
fill in the reference_doc
|
||||
"""
|
||||
claims = frappe.get_all('Expense Claim',
|
||||
filters={'employee':emp,
|
||||
'company': self.justworks_company},
|
||||
@ -170,3 +209,105 @@ class JWImport(Document):
|
||||
return
|
||||
# Hmmm, employee has claims but no large enough unpaid ones
|
||||
# and none with matching amount, so punt.
|
||||
|
||||
def check_data(self, recs):
|
||||
"""Make sure all of the data needed to create entries
|
||||
is available.
|
||||
"""
|
||||
alloc_cache = {}
|
||||
for rec in recs:
|
||||
if not rec.account:
|
||||
rec.data_status = 'No Account'
|
||||
continue
|
||||
if rec.reference == self.collection_ref:
|
||||
rec.data_status = 'OK'
|
||||
continue
|
||||
if not rec.reference_doc:
|
||||
rec.data_status = 'No Party'
|
||||
continue
|
||||
if rec.reference_doctype != 'Employee':
|
||||
rec.data_status = 'OK'
|
||||
continue
|
||||
alloc = get_employee_allocation(rec.reference_doc,
|
||||
rec.invoice_date, alloc_cache)
|
||||
if sum(a.percentage for a in alloc) != 100:
|
||||
rec.data_status = 'Bad Alloc'
|
||||
continue
|
||||
rec.data_status = 'OK'
|
||||
|
||||
def determine_disposition(self, recs):
|
||||
for rec in recs:
|
||||
if rec.data_status != 'OK':
|
||||
rec.dispostion = 'Data Missing'
|
||||
continue
|
||||
|
||||
if rec.reference == self.collection_ref:
|
||||
rec.disposition = 'Collection'
|
||||
elif rec.reference_doctype == 'Expense Claim':
|
||||
rec.disposition = 'Pay Expense'
|
||||
elif rec.reference_doctype == 'Purchase Invoice':
|
||||
rec.disposition = 'Pay Invoice'
|
||||
elif self.jw_obligation_account:
|
||||
rec.disposition = 'Journal'
|
||||
else:
|
||||
rec.disposition = 'Ledger Unset'
|
||||
continue
|
||||
|
||||
if rec.disposition[:3] == 'Pay' and not self.jw_payment_account:
|
||||
rec.disposition = 'Ledger Unset'
|
||||
continue
|
||||
if self.apparent_duplicate(rec):
|
||||
rec.disposition = 'Duplicate'
|
||||
|
||||
def apparent_duplicate(self, rec):
|
||||
if rec.disposition == 'Collection':
|
||||
jes = frappe.get_all('Journal Entry', filters={
|
||||
'company': self.justworks_company,
|
||||
'posting_date': rec.debit_date})
|
||||
if not jes: return False
|
||||
je_names = [je.name for je in jes]
|
||||
return frappe.db.exists({
|
||||
'doctype': 'Journal Entry Account',
|
||||
'parent': ('in', je_names),
|
||||
'account': self.jw_collection_debits,
|
||||
'debit_in_account_currency': rec.amount})
|
||||
if rec.disposition[:3] == 'Pay':
|
||||
pes = frappe.get_all('Payment Entry', filters={
|
||||
'company': self.justworks_company,
|
||||
'paid_from': self.jw_payment_account,
|
||||
'paid_amount': rec.amount})
|
||||
if not pes: return False
|
||||
pe_names = [pe.name for pe in pes]
|
||||
return frappe.db.exists({
|
||||
'doctype': 'Payment Entry Reference',
|
||||
'parent': ('in', pe_names),
|
||||
'reference_name': rec.reference_doc})
|
||||
if rec.disposition == 'Journal':
|
||||
jes = frappe.get_all('Journal Entry', filters={
|
||||
'company': self.justworks_company,
|
||||
'posting_date': rec.debit_date})
|
||||
if not jes: return False
|
||||
je_names = [je.name for je in jes]
|
||||
jeas = frappe.get_all('Journal Entry Account',
|
||||
filters={
|
||||
'parent': ('in', je_names),
|
||||
'account': rec.account
|
||||
},
|
||||
fields=['name','parent'])
|
||||
jea_parents= [jea.parent for jea in jeas]
|
||||
return frappe.db.exists({
|
||||
'doctype': 'Journal Entry Account',
|
||||
'parent': ('in', jea_parents),
|
||||
'account': self.jw_obligation_account})
|
||||
# shouldn't get here but just in case
|
||||
return False
|
||||
|
||||
def get_employee_allocation(emp, date, cache):
|
||||
if emp in cache: return cache[emp]
|
||||
alloc = frappe.get_all('JW Employee Allocation',
|
||||
filters={'employee': emp, 'docstatus': 1},
|
||||
fields=['name','percentage','start_date','end_date'])
|
||||
alloc = [r for r in alloc if (not r.start_date or date >= r.start_date)
|
||||
and (not r.end_date or date <= r.end_date)]
|
||||
cache[emp] = alloc
|
||||
return alloc
|
||||
|
||||
@ -111,7 +111,7 @@
|
||||
"fieldtype": "Select",
|
||||
"in_list_view": 1,
|
||||
"label": "Data",
|
||||
"options": "OK\nBad Alloc\nNo Account",
|
||||
"options": "OK\nNo Account\nNo Party\nBad Alloc",
|
||||
"read_only": 1
|
||||
},
|
||||
{
|
||||
@ -124,7 +124,7 @@
|
||||
"fieldtype": "Select",
|
||||
"in_list_view": 1,
|
||||
"label": "Disposition",
|
||||
"options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nDuplicate\nData Missing"
|
||||
"options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nLedger Unset\nDuplicate\nData Missing"
|
||||
},
|
||||
{
|
||||
"fieldname": "voucher_section",
|
||||
@ -186,7 +186,7 @@
|
||||
"index_web_pages_for_search": 1,
|
||||
"istable": 1,
|
||||
"links": [],
|
||||
"modified": "2020-11-18 23:21:36.546404",
|
||||
"modified": "2020-11-19 20:19:32.479219",
|
||||
"modified_by": "Administrator",
|
||||
"module": "Justworks ERPNext Connector",
|
||||
"name": "JW Invoice Line",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user