Files
imporders/main.py

785 lines
31 KiB
Python

import unicodedata
import argparse
import openpyxl
import logging
import json
import re
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from common import jsonrpc2
from common.utils import *
from common.actionflow import Action, ActionFlow
from enum import Enum
from wakepy import keep
from pathlib import Path
from datetime import datetime
from itertools import repeat, count
from urllib3 import PoolManager
parser = argparse.ArgumentParser(description="Order Import")
parser.add_argument('account', type=str, nargs='?')
parser.add_argument('password', type=str, nargs='?')
parser.add_argument('-d', '--directory', type=str, default=str(Path.home().joinpath('Downloads')))
parser.add_argument('-o', '--profile', nargs='+', action='append', required=True)
parser.add_argument('-t', '--timeout', type=int, default=60)
parser.add_argument('-r', '--attempts', type=int, default=3)
parser.add_argument('-i', '--interval', type=int, default=3)
parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])
args = parser.parse_args()
WEBURL = "https://crm.xiaoman.cn/%s"
APIURL = "https://%s.vosfactures.fr"
def main(driver: WebDriver, logger = logging.getLogger('main')):
http = PoolManager()
parameters = vars(args)
class Status(Enum):
IDLE = 0
READY = 1
RUNNING = 3
STANDBY = 4
class Profile:
def __init__(self, name, subdomain, token, remise, person=None, prefix=None, suffix=None):
self.name = name
self.subdomain = subdomain
self.token = token
self.remise = remise
self.person = person
self.prefix = prefix
self.suffix = suffix
def format(self, number: str):
result = ''.join(filter(bool, [self.prefix, number, self.suffix]))
return result
timer = Timer()
options = dict()
profiles: list[Profile] = list()
status = Status.IDLE
def begin(opts: dict, args: dict):
nonlocal options, status
options.update(opts)
status = Status.RUNNING
parameters.update(args)
timer.clear()
timer.start()
def pause():
nonlocal status
status = Status.STANDBY
timer.pause()
def resume():
nonlocal status
status = Status.RUNNING
driver.switch_to.window(driver.current_window_handle)
timer.start()
jsonrpc2.define('begin', begin)
jsonrpc2.define('pause', pause)
jsonrpc2.define('resume', resume)
jsonrpc2.define('status', lambda: status.name)
jsonrpc2.define('uptime', lambda: timer.delta())
try:
for source, profile in zip(parameters.get('profile'), repeat(dict())):
for key, value in map(lambda o: str.split(o, '=', 2), source):
profile[key.lower().strip()] = value.strip()
item = Profile(**profile)
profiles.append(item)
except Exception as e:
logger.critical('Unable to load profiles', exc_info=e)
return 2
try:
manifest = json.dumps(profiles, default=lambda o: vars(o))
driver.get(str(Path('index.html').resolve()))
driver.execute_script(jsonrpc2.prelude(), manifest, parameters)
except Exception as e:
logger.critical('Unable to load starup page', exc_info=e)
return 3
try:
driver.switch_to.new_window('tab')
driver.set_page_load_timeout(parameters.get('timeout'))
driver.get(WEBURL % 'product')
except TimeoutException:
logger.warning('Timeout')
driver.execute_script("window.stop();")
setup(driver, parameters.get('attempts'), parameters.get('timeout'), parameters.get('interval'))
until(lambda x: 'loginProgress' in x.find_element(By.TAG_NAME, "body").get_attribute('class'), watch=False)
logger.info('Waiting for authentication to complete...')
if (account := parameters.get('account')) and (password := parameters.get('password')):
try:
logger.info('Logging in as %s (%s)', str.split(account, '@', 1).pop(0).capitalize(), account)
locate("input.account").send_keys(account)
locate("input#password").send_keys(password)
click("input.agree-checkbox")
click("button.login-btn")
except Exception as e:
logger.critical('Unable to login to %s', account, exc_info=e)
return 3
while True:
try:
locate("#container", wait=False)
logger.info('Done')
break
except:
sleep(parameters.get('interval'))
class ProductInfo:
def __init__(self, file):
self.wb = openpyxl.load_workbook(file, read_only=True)
self.headers = list()
for col in range(1, self.wb.active.max_column + 1):
value = str(self.wb.active.cell(1, col).value)
self.headers.append(value)
def search(self, by: str, needle) -> dict:
a = self.headers.index(by)
b = None
for row in range(1, self.wb.active.max_row + 1):
if self.wb.active.cell(row, a+1).value == needle:
b = row
break
if b is None: return dict()
return { k: self.wb.active.cell(b, c+1).value for c, k in enumerate(self.headers) }
try:
logger.info('Downloading product list...')
click("header .okki-space .okki-space-item:nth-child(1) button")
click(".okki-dropdown button")
sleep(parameters.get('interval'))
click(".okki-modal.product-export-wrap .mm-selector-rendered")
click(".mm-outside.ui-field-selector-popper .selector-area:nth-child(1) button")
click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary")
while True:
try:
click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary")
sleep(parameters.get('interval'))
click(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) button.okki-btn-link", wait=False)
filename = locate(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) > div > div:nth-child(1) span").get_attribute('title')
break
except:
sleep(parameters.get('interval'))
file = Path(parameters.get('directory')).joinpath(filename)
until(lambda _: file.exists(), watch=False)
p = ProductInfo(file)
driver.close()
driver.switch_to.window(driver.window_handles[0])
status = Status.READY
logger.info('Done')
except Exception as e:
logger.critical('Unable to load products', exc_info=e)
return 4
def wait(seconds: float):
while not sleep(seconds):
if status == Status.RUNNING:
break
def ready(driver: WebDriver):
html = driver.find_element(By.TAG_NAME, 'html')
return 'nprogress-busy' in html.get_attribute('class')
def fetch(url: str, method = 'GET', retry = parameters.get('attempts')):
for attempt in range(1, retry + 1):
try:
response = http.request(method, url)
result = response.json()
return result
except Exception as e:
logger.error('Error while fetching data from %s, retrying... (%d)', url, attempt, exc_info=e)
assert attempt < retry
flow = ActionFlow()
profile = None
progress = { 'task': '' }
selection = 0
class Wait(Action):
@classmethod
def prepare(cls):
return True
@classmethod
def perform(cls):
flow.do(cls)
wait(1)
class Cancel(Action):
@classmethod
def prepare(cls):
nonlocal status
status = Status.RUNNING
return True
class Skip(Action):
@classmethod
def prepare(cls):
return True
flow.append(Wait)
flow.allow(Wait)
flow.do(Wait)
flow.append(Cancel)
flow.append(Skip)
jsonrpc2.define('actions', lambda: flow.capabilities())
jsonrpc2.define('cancel', lambda: flow.do(Cancel))
jsonrpc2.define('skip', lambda: flow.do(Skip))
jsonrpc2.define('progress', lambda: progress)
while not wait(1):
try:
progress.clear()
progress['task'] = 'Task 1 of 4'
flow.allow(Cancel, False)
flow.allow(Skip, False)
if options.get('all'):
profile = profiles[selection]
else:
name = options.pop('profile')
profile = next(filter(lambda o: o.name == name, profiles))
for i in range(len(driver.window_handles), 0):
driver.close()
driver.switch_to.window(driver.window_handles[i])
except (IndexError, KeyError):
logger.info('Done')
status = Status.READY
continue
except StopIteration:
logger.error("Invalid profile '%s'", name)
status = Status.STANDBY
continue
base = APIURL % profile.subdomain
data = list()
df = options.get('datefrom')
dt = options.get('dateto')
types = ['vat']
if options.get('avoir'): types.append('correction')
logger.info('Profile selected: %s', profile.name)
logger.info('Date from %s to %s', df, dt)
flow.allow(Cancel)
for page in count(1):
try:
result = fetch(f'{base}/invoices.json?{'&'.join([f'kinds%5B%5D={k}' for k in types])}&api_token={profile.token}&include_positions=true&per_page=25&page={page}&period=more&date_from={df}&date_to={dt}')
if 'message' in result: raise Exception(result['message'])
if not isinstance(result, list): raise TypeError()
if len(result) == 0: break
logger.info('Downloading invoices (%d)', page)
data.extend(result)
flow.react()
wait(parameters.get('interval'))
except Skip:
pass
except Cancel:
status = Status.READY
break
except Exception as e:
logger.error('Error while fetching data from %s', base, exc_info=e)
break
if len(data) == 0:
logger.warning('Server returned empty response')
selection += 1
continue
logger.info('Initializing Workbook...')
progress['task'] = 'Task 2 of 4'
progress['limit'] = len(data)
workbook = openpyxl.Workbook()
sheet = workbook.active
class Record:
def __init__(self, fields: dict[int, str]):
self.headers = fields
self.data = dict()
def clear(self):
self.data.clear()
def __setitem__(self, key, value):
if key not in self.headers: raise KeyError(key)
self.data[key] = value
def __getitem__(self, key):
if key not in self.headers: raise KeyError(key)
return self.data[key]
# Required fields
# See <https://crm.xiaoman.cn/order/importOrder>
record = Record({
1: '订单号',
2: '订单名称',
3: '订单日期',
4: '当前处理人',
5: '业绩归属部门',
6: '客户编号',
7: '币种',
8: '产品名称',
9: '产品编号',
10: '产品型号',
11: '原价',
12: '折扣率',
13: '单价',
14: '数量',
15: '产品描述',
16: 'AVOIR',
})
sheet.append(record.headers)
categories = dict()
clients = dict()
try:
for i, item in enumerate(data, 1):
number: str = item['number']
logger.info('[%d/%d] Preprocessing data for %s', i, len(data), number)
progress['number'] = number
progress['index'] = i-1
if (id := item['category_id']) not in categories:
flow.react()
categories[id] = fetch(f'{base}/categories/{id}.json?api_token={profile.token}')
if (id := item['client_id']) not in clients:
flow.react()
clients[id] = fetch(f'{base}/clients/{id}.json?api_token={profile.token}')
category = categories.get(item['category_id'])
client = clients.get(item['client_id'])
identity = client['external_id'] if client['company'] else profile.person
date: str = item['issue_date']
kind: str = item['kind']
total = float(item['price_net'])
positions: list = item['positions']
for position in positions:
code: str = position['code']
product: str = position['name']
description: str = position['description']
price = float(position['price_net'] or '0')
discount = float(position['discount_percent'] or '0')
quantity = float(position['quantity'] or '0')
record.clear()
record[1] = profile.format(number)
record[2] = number.replace('/', '-')
record[3] = date
record[4] = category['name']
record[5] = profile.name
record[6] = identity
record[7] = 'USD'
match kind:
case 'vat':
record[8] = product
record[9] = (p.search('产品型号', code) or p.search('产品名称', product)).get('产品编号')
record[10] = code
record[11] = '%.2f' % price
record[12] = '%g%%' % discount
record[13] = '%.2f' % (price * (1 - discount / 100))
record[14] = '%g' % quantity
record[15] = description
case 'correction':
record[8] = p.search('产品编号', profile.remise).get('产品名称')
record[9] = profile.remise
record[13] = '0'
record[14] = '0'
record[16] = '%.2f' % total
positions.clear()
if record[9] is None: logger.warning("Could not identify product '%s'", product)
sheet.append(record.data)
flow.react()
except Skip:
pass
except Cancel:
status = Status.READY
continue
except Exception as e:
logger.error('Error while processing data', exc_info=e)
status = Status.STANDBY
continue
try:
filename = f'Order-Import-{profile.name}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}.xlsx'
file = Path(parameters.get('directory')).joinpath(filename)
logger.info('Saving excel file to %s', str(file))
flow.react()
workbook.save(file)
except Skip:
pass
except Cancel:
status = Status.READY
continue
except Exception as e:
logger.error('Error while saving excel file', exc_info=e)
status = Status.STANDBY
continue
try:
logger.info('Uploading...')
progress.clear()
progress['task'] = 'Task 3 of 4'
driver.switch_to.new_window('tab')
driver.get(WEBURL % 'order/importOrder')
click(".product-import-img-box .import-img-radio:nth-child(2) .mm-radio-group > label:nth-child(2) .mm-radio-input", condition=None)
click(".product-import-img-box .mm-selector-rendered")
click(".mm-outside.mm-select-dropdown ul li:nth-child(%d) span" % (1 if options.get('draft') else 6))
flow.react()
locate(".big-file-upload input", wait=False).send_keys(str(file))
click(".product-import-img-footer button")
click(".product-import-img-footer button.mm-button__primary")
while True:
try:
innerText = driver.find_element(By.CSS_SELECTOR, ".img-box-result-title").get_attribute('innerText')
if innerText == '导入完成': break
except:
wait(parameters.get('interval'))
click(".mm-notification-container .mm-icon-close")
click(".product-import-img-footer button.mm-button__primary")
err = locate(".mm-tbody table tbody tr:nth-child(1) td:nth-child(5) .okki-space-item:nth-child(1) button")
if err.get_attribute('disabled') is None:
logger.warning('Incomplete import detected; downloaded 1 related document')
err.click()
sleep(1)
click(".mm-tbody table tbody tr:nth-child(1) td:nth-child(3) a")
driver.switch_to.window(driver.window_handles[2])
until(ready)
logger.info('Done')
except Skip:
pass
except Cancel:
status = Status.READY
continue
except Exception as e:
logger.error('Error while uploading excel file', exc_info=e)
status = Status.STANDBY
continue
class Parse:
def __init__(self, url: str):
from urllib.parse import urlsplit, parse_qs
parts = list(urlsplit(url))
query = parse_qs(parts[3])
self.parts = parts
self.query = json.loads(query['query'][0]) if 'query' in query else dict()
def encode(self, extra=None):
from urllib.parse import urlencode, urlunsplit
query = { 'query': json.dumps(self.query, separators=(',', ':')) }
if extra is not None: query.update(extra)
self.parts[3] = urlencode(query)
return urlunsplit(self.parts)
def get(self, key: str):
return self.query[key]
def set(self, key: str, value):
self.query[key] = value
progress['task'] = 'Task 4 of 4'
progress['limit'] = len(data)
index = 0
attempts = 0
while index < len(data):
try:
item = data[index]
title = re.search(r'O\d+', item['title'])
number = profile.format(item['number'])
positions = item['positions']
opportunity = None
attempts += 1
if attempts > parameters.get('attempts'):
logger.warning('Exhausted all allowed attempts; skipping %s', number)
index += 1
attempts = 0
continue
progress['number'] = number
progress['index'] = index
flow.allow(Cancel)
flow.allow(Skip)
flow.react()
try:
for page in count(1):
url = Parse(driver.current_url)
url.set('page_size', 1)
url.set('page', page)
url.set('query_filters', [{
'field': 'order_no',
'name': '订单号',
'operator': 'match',
'value': number,
'object_name': 'objOrder',
'field_type': 35,
'unit': '',
'array_flag': 0
}])
driver.get(url.encode())
wait(parameters.get('interval'))
flow.react()
url = Parse(driver.current_url)
if url.get('page') != page: raise Exception(number)
wait(parameters.get('interval'))
link = locate(".virtual-list-wrap .vue-recycle-scroller .vue-recycle-scroller__item-wrapper > div:nth-child(1) .cell[data-cci='1'] a", wait=False)
if link.text != number: continue
logger.info('[%d/%d] Processing %s...', index+1, len(data), number)
link.click()
driver.switch_to.window(driver.window_handles[3])
flow.react()
click(".sticky .okki-space-item:nth-child(1) button")
break
except Skip:
index += 1
attempts = 0
continue
except Cancel:
status = Status.READY
break
except NoSuchElementException:
logger.warning("Could not find invoice '%s'; skipping", number)
index += 1
attempts = 0
continue
except Exception as e:
logger.error("Error while looking up invoice '%s'", number, exc_info=e)
status = Status.STANDBY
continue
if title is not None and (match := title[0]):
try:
driver.switch_to.new_window('tab')
url = Parse(WEBURL % 'crm/business/list')
driver.get(url.encode({ 'mode': 'list' }))
wait(parameters.get('interval'))
flow.react()
try: click(".new-wrapper .paas-next-invoice-list-filter-line-wrapper .okki-btn-background-ghost", wait=False)
except: pass
for page in count(1):
url = Parse(WEBURL % 'crm/business/list')
url.set('keyword', match)
url.set('search_field', 'serial_keyword')
url.set('curPage', page)
url.set('pageSize', 1)
driver.get(url.encode({ 'keyword': match, 'search_field': 'serial_keyword' }))
wait(parameters.get('interval'))
flow.react()
url = Parse(driver.current_url)
if url.get('curPage') != page: raise Exception(match)
wait(parameters.get('interval'))
cell = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(3) .ow-serial-read-pretty_ellipsis", wait=False)
if cell.text != match: continue
link = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(6) a", wait=False)
opportunity = unicodedata.normalize('NFKD', link.get_attribute('title'))
break
except Skip:
index += 1
attempts = 0
continue
except Cancel:
status = Status.READY
break
except NoSuchElementException:
logger.warning("Could not find opportunity '%s'; skipping", match)
except Exception as e:
logger.error("Error while looking up opportunity '%s'", match, exc_info=e)
status = Status.STANDBY
continue
finally:
driver.close()
driver.switch_to.window(driver.window_handles[3])
if opportunity is not None:
try:
flow.react()
dropdown = locate("#rc_select_1")
dropdown.clear()
dropdown.send_keys(opportunity)
menu = locate(".okki-select-dropdown")
menuitems = menu.find_elements(By.CSS_SELECTOR, ".rc-virtual-list-holder-inner > div")
for menuitem in menuitems:
if menuitem.get_attribute('label').strip().startswith(opportunity):
click(menuitem)
except Skip:
index += 1
attempts = 0
continue
except Cancel:
status = Status.READY
break
except Exception as e:
logger.warning('Error while selecting opportunity; skipping', exc_info=e)
try:
click(".okki-pagination-options-size-changer")
click(".okki-select-dropdown .rc-virtual-list-holder-inner > div:nth-child(1)")
except:
logger.warning('Unable to setup pagination; this may cause issues')
pids = list()
wrapper = locate(".paas-order-product-list .row-items", condition=None)
class Eureka(Exception): pass
try:
for page in count(1):
flow.react()
tail = next(reversed(pids), None)
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper)
driver.execute_script("arguments[0].scrollTo(arguments[1], arguments[2]);", wrapper, 0, 0)
for item in wrapper.find_elements(By.CSS_SELECTOR, ".row-item"):
if len(pids) >= len(positions): raise Eureka()
div = item.find_element(By.CSS_SELECTOR, ".cell[data-cci='2'] .cell-inner div")
if (serial := int(div.text)) and serial not in pids:
flow.react()
base = (serial - tail) if tail is not None else index
height = int(item.get_attribute('clientHeight'))
offset = (base - 1) * height
driver.execute_script("arguments[0].scrollTo(arguments[1], arguments[2]);", wrapper, 0, offset)
#span = item.find_element(By.CSS_SELECTOR, ".cell[data-cci='3'] .product-info-group-product-info:nth-child(3) span")
#code = span.text[6:]
value = item.find_element(By.CSS_SELECTOR, ".cell[data-cci='4'] input").get_attribute('value')
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper)
driver.execute_script("arguments[0].scroll(arguments[1], arguments[2]);", wrapper, 400, offset)
wait(parameters.get('interval'))
target = item.find_element(By.CSS_SELECTOR, ".cell[data-cci='6'] input")
if (target.get_attribute('value') == '0'):
target.send_keys(Keys.BACKSPACE)
target.send_keys(value)
pids.append(serial)
button = locate(".text-right li.okki-pagination-next button", condition=None)
if button.get_attribute('disabled') is not None and len(pids) < len(positions):
raise Exception('Product list imcomplete; expected %d, got %d', len(pids), len(positions))
flow.react()
click(button)
except Eureka:
pass
except Skip:
index += 1
attempts = 0
continue
except Cancel:
status = Status.READY
break
except Exception as e:
logger.error('Error while modifying invoice', exc_info=e)
status = Status.STANDBY
continue
try: click(".ow-box button.okki-btn-round", wait=False)
except: pass
try:
wait(parameters.get('interval'))
flow.react()
flow.allow(Cancel, False)
flow.allow(Skip, False)
click(".sticky.bottom-0 button.okki-btn-primary", condition=None)
wait(parameters.get('interval'))
except Skip:
pass
except Cancel:
status = Status.READY
break
except Exception as e:
logger.error('Error while saving document', exc_info=e)
status = Status.STANDBY
continue
finally:
driver.close()
driver.switch_to.window(driver.window_handles[2])
index += 1
attempts = 0
except Exception as e:
logger.error('Unexpected error', exc_info=e)
status = Status.STANDBY
try:
driver.close()
driver.switch_to.window(driver.window_handles[1])
driver.close()
driver.switch_to.window(driver.window_handles[0])
selection += 1
except Exception as e:
logger.error('Unexpected error', exc_info=e)
status = Status.STANDBY
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M")
logger = logging.getLogger()
level = logging.getLevelNamesMapping().get(args.log_level, 'INFO')
logger.setLevel(level)
logger.info('Initializing...')
history = jsonrpc2.History()
logger.addHandler(history)
opts = jsonrpc2.Options()
jsonrpc2.define('history', lambda: history.truncate())
jsonrpc2.run(opts)
logger.info('Creating automation instance')
opts = ChromeOptions()
opts.enable_downloads = True
opts.add_argument('--deny-permission-prompts')
opts.add_experimental_option('prefs', { 'download.default_directory': args.directory })
with keep.presenting():
driver = Chrome(options=opts)
status = main(driver)
except Exception as e:
logger.critical('Fatal error', exc_info=e)
status = 1
finally:
driver.quit()
exit(status)