feat: All data from Justworks file read and processed

justworks_erpnext/justworks_erpnext_connector/doctype/jw_import/jw_import.py

@@ -5,7 +5,9 @@
 from __future__ import unicode_literals
 import frappe
 from frappe.model.document import Document
+from frappe.utils import get_datetime
 from frappe.utils.csvutils import read_csv_content
+from frappe.utils.data import flt
 from frappe.utils.dateutils import parse_date
 
 from fnmatch import fnmatch as wildcard_match
@@ -15,33 +17,66 @@ import re
 
 class JWImport(Document):
 	account_mapping_fields = 'charge_type charge_subtype map_to_account'.split()
+	collection_ref = 'Collection Summary'
 
 	def process_invoice_file(self):
+		# See if we should be generating extra collection entries
+		ccr_flag = self.jw_collection_debits and self.jw_collection_credits
 		fcontent = frappe.get_doc('File',
 			dict(file_url=self.load_from_file)).get_content()
 		rows = read_csv_content(fcontent)
 		header = [frappe.scrub(h) for h in rows.pop(0)]
 		self.set('lines', [])
+		inv_total = 0.0
+		id_idx = header.index('invoice_id')
+		cur_id = ''
+		rows.append(['THIS_IS_END_SENTINEL'] * (id_idx+1))
 		# Load the base data from the file
 		for row in rows:
-			rec = self.append('lines', {})
+			# Emit a collection record if invoice is complete
+			if ccr_flag and cur_id and cur_id != row[id_idx]:
+				lrec = self.lines[-1]
+				ctmp = dict(invoice_id=cur_id,
+					invoice_date=lrec.invoice_date,
+					debit_date=lrec.debit_date,
+					reference=self.collection_ref,
+					amount=inv_total)
+				crec = self.append('lines', {})
+				for k in ctmp: crec.set(k, ctmp[k])
+				inv_total = 0.0
+			if row[id_idx] == 'THIS_IS_END_SENTINEL': break
+
+			temp = {}
 			for fieldname, val in zip(header, row):
 				if val:
 					if fieldname[-4:] == 'date':
-						val = parse_date(val)
+						val = get_datetime(parse_date(val)).date()
 					elif fieldname == 'amount':
 						val = re.sub('^[^.,\d]*', '', val)
-				rec.set(fieldname, val)
+				temp[fieldname] = val
+			if (self.start_invoice_date and temp['invoice_date'] < self.start_invoice_date):
+				continue
+			if (self.end_invoice_date and temp['invoice_date'] > self.end_invoice_date):
+				continue
+			cur_id = temp['invoice_id']
+			rec = self.append('lines', {})
+			for k in temp:
+				rec.set(k, temp[k])
+			inv_total += flt(rec.amount)
+
 		# Now determine all of the derived data
 		self.map_accounts(self.lines)
 		self.determine_references(self.lines)
-		#self.check_data(self.lines)
+		self.check_data(self.lines)
-		#self.determine_disposition(self.lines)
+		self.determine_disposition(self.lines)
 
 	def map_accounts(self, recs):
 		amf = self.account_mapping_fields
 		acc_maps = frappe.get_all('JW Account Mapping', fields=amf)
 		for rec in recs:
+			if rec.reference == self.collection_ref:
+				rec.set('account', self.jw_collection_debits)
+				continue
 			for amap in acc_maps:
 				if all(wildcard_match((rec.get(fld,'') or ''),
 							amap.get(fld))
@@ -82,9 +117,10 @@ class JWImport(Document):
 			# Ok, this is an expense claim:
 			self.search_expense_claim(rec, emp)
 
-	# Look for and return an employee matching the reference of the given
-	# record, if any
 	def search_employee(self, rec):
+		"""Look for and return an employee matching the reference
+		of the given record, if any
+		"""
 		fullref = rec.reference
 		(last, comma, restref) = fullref.partition(',')
 		emps = (frappe.get_all('Employee',
@@ -100,9 +136,10 @@ class JWImport(Document):
 		closest = max(emps, key=lambda e: len(commonprefix(e, first)))
 		return closest.name
 
-	# Look for a purchase invoice with the same supplier and amount
-	# as rec and fill in the reference_doc if found
 	def search_invoice(self, rec):
+		"""Look for a purchase invoice with the same supplier and
+		amount as rec and fill in the reference_doc if found
+		"""
 		# Reference should be a supplier
 		suppliers = (frappe.get_all('Supplier',
 				{'supplier_name': rec.reference}) or
@@ -140,9 +177,11 @@ class JWImport(Document):
 		# enough left to be paid and none with matching
 		# amount, so punt.
 
-	# Look for an expense claim with given emp and the same amount as
-	# the given rec, and if one is found, fill in the reference_doc
 	def search_expense_claim(self, rec, emp):
+		"""Look for an expense claim with given emp and the same
+		amount as the given rec, and if one is found,
+		fill in the reference_doc
+		"""
 		claims = frappe.get_all('Expense Claim',
 			filters={'employee':emp,
 				'company': self.justworks_company},
@@ -170,3 +209,105 @@ class JWImport(Document):
 			return
 		# Hmmm, employee has claims but no large enough unpaid ones
 	        # and none with matching amount, so punt.
+
+	def check_data(self, recs):
+		"""Make sure all of the data needed to create entries
+		is available.
+		"""
+		alloc_cache = {}
+		for rec in recs:
+			if not rec.account:
+				rec.data_status = 'No Account'
+				continue
+			if rec.reference == self.collection_ref:
+				rec.data_status = 'OK'
+				continue
+			if not rec.reference_doc:
+				rec.data_status = 'No Party'
+				continue
+			if rec.reference_doctype != 'Employee':
+				rec.data_status = 'OK'
+				continue
+			alloc = get_employee_allocation(rec.reference_doc,
+				rec.invoice_date, alloc_cache)
+			if sum(a.percentage for a in alloc) != 100:
+				rec.data_status = 'Bad Alloc'
+				continue
+			rec.data_status = 'OK'
+
+	def determine_disposition(self, recs):
+		for rec in recs:
+			if rec.data_status != 'OK':
+				rec.dispostion = 'Data Missing'
+				continue
+
+			if rec.reference == self.collection_ref:
+				rec.disposition = 'Collection'
+			elif rec.reference_doctype == 'Expense Claim':
+				rec.disposition = 'Pay Expense'
+			elif rec.reference_doctype == 'Purchase Invoice':
+				rec.disposition = 'Pay Invoice'
+			elif self.jw_obligation_account:
+				rec.disposition = 'Journal'
+			else:
+				rec.disposition = 'Ledger Unset'
+				continue
+
+			if rec.disposition[:3] == 'Pay' and not self.jw_payment_account:
+				rec.disposition = 'Ledger Unset'
+				continue
+			if self.apparent_duplicate(rec):
+				rec.disposition = 'Duplicate'
+
+	def apparent_duplicate(self, rec):
+		if rec.disposition == 'Collection':
+			jes = frappe.get_all('Journal Entry', filters={
+				'company': self.justworks_company,
+				'posting_date': rec.debit_date})
+			if not jes: return False
+			je_names = [je.name for je in jes]
+			return frappe.db.exists({
+				'doctype': 'Journal Entry Account',
+				'parent': ('in', je_names),
+				'account': self.jw_collection_debits,
+				'debit_in_account_currency': rec.amount})
+		if rec.disposition[:3] == 'Pay':
+			pes = frappe.get_all('Payment Entry', filters={
+				'company': self.justworks_company,
+				'paid_from': self.jw_payment_account,
+				'paid_amount': rec.amount})
+			if not pes: return False
+			pe_names = [pe.name for pe in pes]
+			return frappe.db.exists({
+				'doctype': 'Payment Entry Reference',
+				'parent': ('in', pe_names),
+				'reference_name': rec.reference_doc})
+		if rec.disposition == 'Journal':
+			jes = frappe.get_all('Journal Entry', filters={
+				'company': self.justworks_company,
+				'posting_date': rec.debit_date})
+			if not jes: return False
+			je_names = [je.name for je in jes]
+			jeas = frappe.get_all('Journal Entry Account',
+				filters={
+					'parent': ('in', je_names),
+					'account': rec.account
+				},
+				fields=['name','parent'])
+			jea_parents= [jea.parent for jea in jeas]
+			return frappe.db.exists({
+				'doctype': 'Journal Entry Account',
+				'parent': ('in', jea_parents),
+				'account': self.jw_obligation_account})
+		# shouldn't get here but just in case
+		return False
+
+def get_employee_allocation(emp, date, cache):
+	if emp in cache: return cache[emp]
+	alloc = frappe.get_all('JW Employee Allocation',
+		filters={'employee': emp, 'docstatus': 1},
+		fields=['name','percentage','start_date','end_date'])
+	alloc = [r for r in alloc if (not r.start_date or date >= r.start_date)
+		and (not r.end_date or date <= r.end_date)]
+	cache[emp] = alloc
+	return alloc

justworks_erpnext/justworks_erpnext_connector/doctype/jw_invoice_line/jw_invoice_line.json

@@ -111,7 +111,7 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Data",
-   "options": "OK\nBad Alloc\nNo Account",
+   "options": "OK\nNo Account\nNo Party\nBad Alloc",
    "read_only": 1
   },
   {
@@ -124,7 +124,7 @@
    "fieldtype": "Select",
    "in_list_view": 1,
    "label": "Disposition",
-   "options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nDuplicate\nData Missing"
+   "options": "Journal\nPay Invoice\nPay Expense\nCollection\nIgnore\nLedger Unset\nDuplicate\nData Missing"
   },
   {
    "fieldname": "voucher_section",
@@ -186,7 +186,7 @@
  "index_web_pages_for_search": 1,
  "istable": 1,
  "links": [],
- "modified": "2020-11-18 23:21:36.546404",
+ "modified": "2020-11-19 20:19:32.479219",
  "modified_by": "Administrator",
  "module": "Justworks ERPNext Connector",
  "name": "JW Invoice Line",