An app for Frappe framework that serves as an add-on to the ERPNext app to allow Justworks invoices to be loaded as Journal Entry documents into ERPNext.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

472 lines
16 KiB

# -*- coding: utf-8 -*-
# Copyright (c) 2020, Studio Infinity and contributors
# For license information, please see license.txt
from __future__ import unicode_literals
import frappe
from frappe.model.document import Document
from frappe.utils import get_datetime
from frappe.utils.csvutils import read_csv_content
from frappe.utils.data import flt
from frappe.utils.dateutils import parse_date
from erpnext.accounts.doctype.accounting_dimension.accounting_dimension import get_accounting_dimensions
from collections import defaultdict
from fnmatch import fnmatch as wildcard_match
import json
from operator import itemgetter
from os.path import commonprefix
import re
class JWImport(Document):
account_mapping_fields = 'charge_type charge_subtype map_to_account'.split()
jea_remark_fields = 'voucher_id voucher_type charge_subtype department reference_work_id notes'.split()
je_remark_fields = 'voucher_pay_date period_start_date period_end_date'.split()
collection_ref = 'Collection Summary'
def process_invoice_file(self):
# See if we should be generating extra collection entries
ccr_flag = self.jw_collection_debits and self.jw_collection_credits
fcontent = frappe.get_doc('File',
dict(file_url=self.load_from_file)).get_content()
rows = read_csv_content(fcontent)
header = [frappe.scrub(h) for h in rows.pop(0)]
self.set('lines', [])
inv_total = 0.0
id_idx = header.index('invoice_id')
cur_id = ''
rows.append(['THIS_IS_END_SENTINEL'] * (id_idx+1))
si_date = ''
if self.start_invoice_date:
si_date = get_datetime(self.start_invoice_date).date()
ei_date = ''
if self.end_invoice_date:
ei_date = get_datetime(self.end_invoice_date).date()
# Load the base data from the file
for row in rows:
# Emit a collection record if invoice is complete
if ccr_flag and cur_id and cur_id != row[id_idx]:
lrec = self.lines[-1]
factor = 10.0**lrec.precision('amount')
inv_total = int(inv_total*factor + 0.5)/factor
ctmp = dict(invoice_id=cur_id,
invoice_date=lrec.invoice_date,
debit_date=lrec.debit_date,
reference=self.collection_ref,
amount=inv_total)
crec = self.append('lines', {})
for k in ctmp: crec.set(k, ctmp[k])
inv_total = 0.0
cur_id = ''
if row[id_idx] == 'THIS_IS_END_SENTINEL': break
temp = {}
for fieldname, val in zip(header, row):
if val:
if fieldname[-4:] == 'date':
val = get_datetime(parse_date(val)).date()
elif fieldname == 'amount':
val = re.sub('^[^-.,\d]*', '', val)
temp[fieldname] = val
if (si_date and temp['invoice_date'] < si_date):
continue
if (ei_date and temp['invoice_date'] > ei_date):
continue
cur_id = temp['invoice_id']
rec = self.append('lines', {})
for k in temp:
rec.set(k, temp[k])
inv_total += flt(rec.amount)
# Now determine all of the derived data
self.map_accounts(self.lines)
self.determine_references(self.lines)
self.check_data(self.lines)
self.determine_disposition(self.lines)
def map_accounts(self, recs):
amf = self.account_mapping_fields
acc_maps = frappe.get_all('JW Account Mapping', fields=amf)
for rec in recs:
if rec.reference == self.collection_ref:
rec.set('account', self.jw_collection_debits)
continue
for amap in acc_maps:
if all(wildcard_match((rec.get(fld,'') or ''),
amap.get(fld))
for fld in amf[:-1]):
rec.set('account', amap.get(amf[-1]))
break
def determine_references(self, recs):
expense_acct = frappe.get_value('Company',
self.justworks_company,
'default_expense_claim_payable_account')
# First determine the reference type for each record
for rec in recs:
acct = rec.get('account', '')
if not acct: continue
if acct == expense_acct:
rec.set('reference_doctype', 'Expense Claim')
continue
if frappe.get_value('Account', acct, 'account_type') == 'Payable':
rec.set('reference_doctype', 'Purchase Invoice')
continue
if frappe.get_value('Account', acct, 'root_type') == 'Expense':
rec.set('reference_doctype', 'Employee')
# Now try to determine the matching doc for each record
for rec in recs:
doctype = rec.get('reference_doctype', '')
if not doctype or not rec.get('reference', ''):
continue
if doctype == 'Purchase Invoice':
self.search_invoice(rec)
continue
emp = self.search_employee(rec)
if not emp: continue
if doctype == 'Employee':
rec.set('reference_doc', emp)
continue
# Ok, this is an expense claim:
self.search_expense_claim(rec, emp)
def search_employee(self, rec):
"""Look for and return an employee matching the reference
of the given record, if any
"""
fullref = rec.reference
(last, comma, restref) = fullref.partition(',')
emps = (frappe.get_all('Employee',
filters={'justworks_name': fullref})
or frappe.get_all('Employee',
filters={'last_name': last},
fields=['name', 'first_name']))
if not emps: return ''
if len(emps) == 1: return emps[0].name
# We have multiple employees with same last name, take
# best match for first name
first = restref.partition(' ')[0]
closest = max(emps, key=lambda e: len(commonprefix(e, first)))
return closest.name
def search_invoice(self, rec):
"""Look for a purchase invoice with the same supplier and
amount as rec and fill in the reference_doc if found
"""
fullref = rec.reference
(lastref, comma, restref) = fullref.partition(',')
# Reference should be a supplier
suppliers = (frappe.get_all('Supplier',
{'supplier_name': fullref})
or frappe.get_all('Supplier', {
'supplier_name': ['like', f"%{fullref}%"]})
or (comma and frappe.get_all('Supplier', {
'supplier_name': ['like', f"%{lastref}"]}))
or (comma and frappe.get_all('Supplier', {
'supplier_name': ['like', f"%{lastref}%"]})))
if not suppliers or len(suppliers) != 1: return
suplr = suppliers[0].name
invoices = frappe.get_all('Purchase Invoice',
filters={'supplier': suplr,
'posting_date': ['<=', rec.debit_date],
'company': self.justworks_company},
fields=['name', 'posting_date as date', 'grand_total',
'rounded_total', 'outstanding_amount'])
if not invoices: return
if len(invoices) == 1:
rec.set('reference_doc', invoices[0].name)
return
rec_amt= flt(rec.amount)
match_inv = ([inv for inv in invoices
if inv.outstanding_amount == rec_amt]
or [inv for inv in invoices
if inv.outstanding_amount > rec_amt])
if len(match_inv) > 0:
# Best match is earliest unpaid invoice
earliest = min(match_inv, key=itemgetter('date'))
rec.set('reference_doc', earliest.name)
return
match_inv = [inv for inv in invoices
if inv.grand_total == rec_amt or
inv.rounded_total == rec_amt]
if len(match_inv) > 0:
latest = max(match_inv, key=itemgetter('date'))
rec.set('reference_doc', latest.name)
# Hmmm, supplier has invoices but none with
# enough left to be paid and none with matching
# amount, so punt.
def search_expense_claim(self, rec, emp):
"""Look for an expense claim with given emp and the same
amount as the given rec, and if one is found,
fill in the reference_doc
"""
claims = frappe.get_all('Expense Claim',
filters={'employee':emp,
'company': self.justworks_company},
fields=['name', 'total_amount_reimbursed as total',
'is_paid', 'posting_date as date'])
if not claims: return
if len(claims) == 1:
rec.set('reference_doc', claims[0].name)
return
rec_amt = flt(rec.amount)
match_claims = {c for c in claims if c.total == rec_amt}
unpaid_claims = {c for c in claims
if c.total >= rec_amt and not c.is_paid}
mu_claims = match_claims & unpaid_claims
if len(mu_claims) > 0:
earliest = min(mu_claims, key=itemgetter('date'))
rec.set('reference_doc', earliest.name)
return
if len(match_claims) > 0:
latest = max(match_claims, key=itemgetter('date'))
rec.set('reference_doc', latest.name)
return
if len(unpaid_claims) > 0:
earliest = min(unpaid_claims, key=itemgetter('date'))
rec.set('reference_doc', earliest.name)
return
# Hmmm, employee has claims but no large enough unpaid ones
# and none with matching amount, so punt.
def check_data(self, recs):
"""Make sure all of the data needed to create entries
is available.
"""
alloc_cache = {}
for rec in recs:
if not rec.account:
rec.data_status = 'No Account'
continue
if rec.reference == self.collection_ref:
rec.data_status = 'OK'
continue
if not rec.reference_doc:
rec.data_status = 'No Party'
continue
if rec.reference_doctype != 'Employee':
rec.data_status = 'OK'
continue
alloc = get_employee_allocation(rec.reference_doc,
rec.invoice_date, alloc_cache)
if sum(a.percentage for a in alloc) != 100:
rec.data_status = 'Bad Alloc'
continue
rec.data_status = 'OK'
def determine_disposition(self, recs):
for rec in recs:
if rec.data_status != 'OK':
rec.disposition = 'Data Missing'
continue
if abs(flt(rec.amount)) < 0.01:
rec.disposition = 'Ignore'
continue
if rec.reference == self.collection_ref:
rec.disposition = 'Collection'
elif rec.reference_doctype == 'Expense Claim':
rec.disposition = 'Pay Expense'
elif rec.reference_doctype == 'Purchase Invoice':
rec.disposition = 'Pay Invoice'
elif self.jw_obligation_account:
rec.disposition = 'Journal'
else:
rec.disposition = 'Ledger Unset'
continue
if rec.disposition[:3] == 'Pay' and not self.jw_payment_account:
rec.disposition = 'Ledger Unset'
continue
if self.apparent_duplicate(rec):
rec.disposition = 'Duplicate'
def apparent_duplicate(self, rec):
if rec.disposition == 'Collection':
jes = frappe.get_all('Journal Entry', filters={
'company': self.justworks_company,
'docstatus': 1,
'posting_date': rec.debit_date})
if not jes: return False
je_names = [je.name for je in jes]
return frappe.db.exists({
'doctype': 'Journal Entry Account',
'parent': ('in', je_names),
'account': self.jw_collection_debits,
'debit_in_account_currency': flt(rec.amount)})
if rec.disposition[:3] == 'Pay':
pes = frappe.get_all('Payment Entry', filters={
'company': self.justworks_company,
'docstatus': 1,
'paid_from': self.jw_payment_account,
'paid_amount': rec.amount})
if not pes: return False
pe_names = [pe.name for pe in pes]
return frappe.db.exists({
'doctype': 'Payment Entry Reference',
'parent': ('in', pe_names),
'reference_name': rec.reference_doc})
if rec.disposition == 'Journal':
jes = frappe.get_all('Journal Entry', filters={
'company': self.justworks_company,
'docstatus': 1,
'posting_date': rec.invoice_date})
if not jes: return False
je_names = [je.name for je in jes]
jeas = frappe.get_all('Journal Entry Account',
filters={
'parent': ('in', je_names),
'account': rec.account
},
fields=['name','parent'])
if not jeas: return False
jea_parents= [jea.parent for jea in jeas]
return frappe.db.exists({
'doctype': 'Journal Entry Account',
'parent': ('in', jea_parents),
'account': self.jw_obligation_account})
# shouldn't get here but just in case
return False
def get_employee_allocation(emp, date, cache):
if emp in cache: return cache[emp]
alloc = frappe.get_all('JW Employee Allocation',
filters={'employee': emp, 'docstatus': 1},
fields=['name','percentage','start_date','end_date'])
alloc = [frappe.get_doc('JW Employee Allocation',r.name) for r
in alloc if (not r.start_date or date >= r.start_date)
and (not r.end_date or date <= r.end_date)]
cache[emp] = alloc
return alloc
@frappe.whitelist()
def record_entries(doc):
importer = frappe.get_doc(json.loads(doc))
n_detail = len(importer.lines)
n_entries = 0
last_l = n_detail - 1
current_jeas = []
current_total = 0.0
acc_dims = get_accounting_dimensions()
acc_dims.append('cost_center')
rebalance_dims = [dim for dim in acc_dims if importer.get(dim,'')]
current_dim_imbalance = defaultdict(float)
standard_dims = {dim:importer.get(dim,'') for dim in acc_dims}
standard_alloc = [dict(standard_dims, percentage=100)]
standard_remark = 'Transfer to ' + ",".join(f"{d} {v}"
for (d,v) in standard_dims.items())
# The following makes a tuple from a comprehension:
standard_key = *((dim,standard_dims[dim]) for dim in rebalance_dims),
alloc_cache = {}
je_remarks = {field:'' for field in importer.je_remark_fields}
for l in range(n_detail):
rec = importer.lines[l]
if rec.disposition == 'Journal':
rec_amt =flt(rec.amount)
current_total += rec_amt
for field in importer.je_remark_fields:
if rec.get(field,''): je_remarks[field] = rec.get(field)
jea_remarks = "\n".join(f"{f}: {rec.get(f)}"
for f in importer.jea_remark_fields
if rec.get(f,''))
alloc = standard_alloc
if rec.reference_doctype == 'Employee':
alloc = get_employee_allocation(
rec.reference_doc,
get_datetime(rec.invoice_date).date(),
alloc_cache)
so_far = 0.0
for s in range(len(alloc)-1, -1, -1):
share = alloc[s]
amnt = int(rec_amt*share.percentage)/100.0
if s == 0: amnt = rec_amt - so_far
so_far += amnt
cred = 0.0; deb = amnt;
if (amnt < 0.0):
cred = -amnt; deb = 0.0;
if importer.dim_balance_account:
imb_key = *((dim,share.get(dim))
for dim in rebalance_dims),
if imb_key != standard_key:
current_dim_imbalance[imb_key] += amnt
current_jeas.append(dict(
{d:share.get(d) for d in acc_dims},
account=rec.account,
party_type=rec.reference_doctype,
party=rec.reference_doc,
account_currency=importer.justworks_currency,
debit_in_account_currency=deb,
credit_in_account_currency=cred,
user_remark=jea_remarks))
rec.set('disposition', 'Duplicate')
# Should we emit a journal entry?
if (current_total > 0.0 and (l == last_l
or importer.lines[l+1].invoice_id != rec.invoice_id)):
je_remarks = "\n".join(f"{f}: {je_remarks[f]}"
for f in importer.je_remark_fields
if je_remarks[f])
je = frappe.get_doc(doctype='Journal Entry',
title='Justworks Invoice '+rec.invoice_date,
voucher_type='Journal Entry',
company=importer.justworks_company,
posting_date=rec.invoice_date,
cheque_no=rec.invoice_id,
cheque_date=rec.invoice_date,
user_remark=je_remarks,
pay_to_recd_from='Justworks')
for jeaf in current_jeas:
jea = je.append('accounts', {})
for f in jeaf: jea.set(f, jeaf[f])
# Now generate "other sides":
if importer.dim_balance_account:
for dimvec in current_dim_imbalance:
amnt = current_dim_imbalance[dimvec]
if amnt == 0.0: continue
jea = je.append('accounts', {})
jea.set('account',
importer.dim_balance_account)
for (dim, val) in dimvec:
jea.set(dim, val)
jea.set('account_currency',
importer.justworks_currency)
jea.set('debit_in_account_currency',
0.0 if amnt > 0.0 else -amnt)
jea.set('credit_in_account_currency',
amnt if amnt > 0.0 else 0.0)
jea.set('user_remark', standard_remark)
oja = je.append('accounts', {})
oja.set('account',
importer.dim_balance_account)
for (dim,val) in standard_dims.items():
oja.set(dim,val)
oja.set('account_currency',
importer.justworks_currency)
oja.set('debit_in_account_currency',
amnt if amnt > 0.0 else 0.0)
oja.set('credit_in_account_currency',
0.0 if amnt > 0.0 else -amnt)
oja.set('user_remark', standard_remark)
bja = je.append('accounts', {})
bja.set('account', importer.jw_obligation_account)
for (dim,val) in standard_dims.items():
bja.set(dim,val)
bja.set('account_currency', importer.justworks_currency)
bja.set('debit_in_account_currency',
0.0 if current_total > 0.0 else -current_total)
bja.set('credit_in_account_currency',
current_total if current_total > 0.0 else 0.0)
bja.set('user_remark', 'Total Justworks expenses')
je.insert()
if importer.auto_submit: je.submit()
n_entries += 1
current_jeas = []
current_total = 0.0
for k in current_dim_imbalance:
current_dim_imbalance[k] = 0.0
je_remarks = {field:'' for field
in importer.je_remark_fields}
importer.save()
frappe.msgprint(f"Entries created: {n_entries}", "Justworks Import");