Files
imporders/main.py
T
2026-08-11 23:24:43 +08:00

775 lines
32 KiB
Python

import argparse
import openpyxl
import logging
import json
import re
from selenium.webdriver import Chrome, ChromeOptions
from selenium.webdriver.remote.webdriver import WebDriver
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from common.utils import *
from common.timer import Timer
from common.jsonrpc2 import ServiceProvider
from common.actionflow import Action, ActionFlow
from enum import Enum
from wakepy import keep
from pathlib import Path
from datetime import datetime
from itertools import count
from urllib3 import PoolManager
parser = argparse.ArgumentParser(description="Order Import")
parser.add_argument('account', type=str, nargs='?')
parser.add_argument('password', type=str, nargs='?')
parser.add_argument('-d', '--directory', type=str, default=str(Path.home().joinpath('Downloads')))
parser.add_argument('-o', '--profile', nargs='+', action='append', required=True)
parser.add_argument('-t', '--timeout', type=int, default=60)
parser.add_argument('-r', '--attempts', type=int, default=3)
parser.add_argument('-i', '--interval', type=int, default=3)
parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'])
args = parser.parse_args()
WEBURL = "https://crm.xiaoman.cn/%s"
APIURL = "https://%s.vosfactures.fr"
def main(driver: WebDriver, logger = logging.getLogger('main')):
parameters = vars(args)
http = PoolManager()
sp = ServiceProvider(ServiceProvider.Options(application='Imporders'))
class Status(Enum):
BUSY = 0
READY = 1
RUNNING = 2
STANDBY = 3
class Profile:
def __init__(self, name, subdomain, token, remise, person=None, prefix=None, suffix=None):
self.name = name
self.subdomain = subdomain
self.token = token
self.remise = remise
self.person = person
self.prefix = prefix
self.suffix = suffix
def format(self, number: str):
result = ''.join(filter(bool, [self.prefix, number, self.suffix]))
return result
t1 = Timer()
t2 = Timer()
options = dict()
status = Status.BUSY
def begin(opts: dict, args: dict):
nonlocal status
options.update(opts)
status = Status.RUNNING
parameters.update(args)
t1.clear()
t1.start()
def pause():
nonlocal status
status = Status.STANDBY
t1.pause()
t2.pause()
def resume():
nonlocal status
status = Status.RUNNING
driver.switch_to.window(driver.current_window_handle)
t1.start()
t2.start()
sp.add(begin, pause, resume)
sp.set('status', lambda: status.name)
sp.set('uptime', lambda: [t1.delta()])
try:
profiles = [
Profile(**{ k.lower().strip(): v.strip() for k, v in map(lambda o: str.split(o, '=', 2), p) })
for p in parameters['profile']
]
except Exception as e:
logger.critical('Unable to load profiles', exc_info=e)
return 2
try:
sp.set('context', lambda: { 'profiles': list(map(vars, profiles)), 'parameters': parameters })
driver.get(sp.run())
except Exception as e:
logger.critical('Unable to load starup page', exc_info=e)
return 3
try:
driver.switch_to.new_window('tab')
driver.set_page_load_timeout(parameters['timeout'])
driver.get(WEBURL % 'product')
except TimeoutException:
logger.warning('Timeout')
driver.execute_script("window.stop();")
setup(driver, parameters)
until(lambda x: 'loginProgress' in x.find_element(By.TAG_NAME, "body").get_attribute('class'), watch=False)
logger.info('Waiting for authentication...')
if (account := parameters['account']) and (password := parameters['password']):
try:
logger.info('Logging in as %s (%s)', str.split(account, '@', 1).pop(0).capitalize(), account)
locate("input.account").send_keys(account)
locate("input#password").send_keys(password)
click("input.agree-checkbox")
click("button.login-btn")
except Exception as e:
logger.critical('Unable to login to %s', account, exc_info=e)
return 3
while True:
try:
locate("#container", wait=False)
logger.info('Done')
break
except:
sleep(parameters['interval'])
class ProductInfo:
def __init__(self, file):
self.wb = openpyxl.load_workbook(file, read_only=True)
self.indices = dict()
self.headers = dict()
def index(self, *keys: str):
self.indices.clear()
self.headers.clear()
for col in range(1, self.wb.active.max_column + 1):
value = str(self.wb.active.cell(1, col).value)
self.headers[value] = col
for row in range(1, self.wb.active.max_row + 1):
for key, col in map(lambda k: (k, self.headers[k]), keys):
value = str(self.wb.active.cell(row, col).value)
self.indices.setdefault(key, dict())[value] = row
def at(self, axis: str, index) -> dict:
if (a := self.indices.get(axis)) is None or (b := dict.get(a, index)) is None: return dict()
return { k: self.wb.active.cell(b, c).value for k, c in self.headers.items() }
try:
logger.info('Downloading product list...')
click("header .okki-space .okki-space-item:nth-child(1) button")
click(".okki-dropdown button")
sleep(parameters['interval'])
click(".okki-modal.product-export-wrap .mm-selector-rendered")
click(".mm-outside.ui-field-selector-popper .selector-area:nth-child(1) button")
click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary")
while True:
try:
click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary")
sleep(parameters['interval'])
click(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) button.okki-btn-link", wait=False)
filename = locate(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) > div > div:nth-child(1) span").get_attribute('title')
break
except:
sleep(parameters['interval'])
file = Path(parameters['directory']).joinpath(filename)
until(lambda _: file.exists(), watch=False)
p = ProductInfo(file)
driver.close()
driver.switch_to.window(driver.window_handles[0])
logger.info('Indexing product information...')
p.index('产品编号', '产品名称', '产品型号')
logger.info('Done')
status = Status.READY
except Exception as e:
logger.critical('Unable to load products', exc_info=e)
return 4
def fetch(url: str, method = 'GET', retry = parameters['attempts']):
for attempt in range(1, retry + 1):
try:
response = http.request(method, url)
result = response.json()
return result
except Exception as e:
logger.warning('Error while fetching data from %s, retrying... (%d)', url, attempt, exc_info=e)
assert attempt < retry, "Exceeded maximum retry attempts"
class Wait(Action):
@classmethod
def prepare(cls):
return True
@classmethod
def perform(cls):
if status == Status.RUNNING: return False
sleep(0.2); return True
class Sleep(Action):
@classmethod
def prepare(cls):
return True
@classmethod
def perform(cls):
sleep(parameters['interval'])
return False
class Cancel(Action):
@classmethod
def prepare(cls):
nonlocal status
status = Status.RUNNING
return True
@classmethod
def perform(cls):
nonlocal status
status = Status.READY
driver.switch_to.window(driver.window_handles[0])
raise cls
class Skip(Action):
@classmethod
def prepare(cls):
return True
@classmethod
def perform(cls):
driver.switch_to.window(driver.current_window_handle)
raise cls
flow = ActionFlow()
flow.stage(Wait, Sleep, Cancel, Skip)
profile = None
progress = { 'task': '' }
selection = 0
sp.add(pairs=[ (k.lower(), v) for k, v in flow ])
sp.set('actions', lambda: flow.capabilities())
sp.set('progress', lambda: progress)
while not flow.react(Wait):
try:
for i in range(len(driver.window_handles), 1, -1):
driver.switch_to.window(driver.window_handles[i-1])
driver.close()
driver.switch_to.window(driver.window_handles[i-2])
if options.get('all'):
profile = profiles[selection]
selection += 1
else:
name = options.pop('profile')
profile = next(filter(lambda o: o.name == name, profiles))
except (IndexError, KeyError):
logger.info('Done')
status = Status.READY
continue
except StopIteration:
logger.error("Invalid profile '%s'", name)
status = Status.STANDBY
continue
except Exception as e:
logger.error('Unexpected error', exc_info=e)
status = Status.STANDBY
continue
progress.clear()
progress['task'] = 'Task 1 of 4'
t2.clear()
t2.start()
sp.pop('uptime')
sp.set('uptime', lambda: [t1.delta(), t2.delta()])
base = APIURL % profile.subdomain
data = list()
df = options.get('datefrom')
dt = options.get('dateto')
types = ['vat']
if options.get('avoir'): types.append('correction')
logger.info('Profile selected: %s', profile.name)
logger.info('Date from %s to %s', df, dt)
flow.allow(Cancel)
flow.deter(Skip)
for page in count(1):
try:
result = fetch(f'{base}/invoices.json?{'&'.join([f'kinds%5B%5D={k}' for k in types])}&api_token={profile.token}&include_positions=true&per_page=25&page={page}&period=more&date_from={df}&date_to={dt}')
if 'message' in result: raise Exception(result['message'])
if not isinstance(result, list): raise TypeError()
if len(result) == 0: break
logger.info('Downloading invoices (%d)', page)
data.extend(result)
flow.react(Wait, Sleep)
except Skip:
pass
except Cancel:
break
except Exception as e:
logger.error('Error while fetching data from %s', base, exc_info=e)
break
if len(data) == 0:
logger.warning('Server returned an empty response')
continue
logger.info('Initializing Workbook...')
progress['task'] = 'Task 2 of 4'
progress['limit'] = len(data)
progress['index'] = 0
t2.clear()
t2.start()
workbook = openpyxl.Workbook()
sheet = workbook.active
class Record:
def __init__(self, fields: dict[int, str]):
self.headers = fields
self.data = dict()
def clear(self):
self.data.clear()
def __setitem__(self, key, value):
if key not in self.headers: raise KeyError(key)
self.data[key] = value
def __getitem__(self, key):
if key not in self.headers: raise KeyError(key)
return self.data[key]
# Required fields
# See <https://crm.xiaoman.cn/order/importOrder>
record = Record({
1: '订单号',
2: '订单名称',
3: '订单日期',
4: '当前处理人',
5: '业绩归属部门',
6: '客户编号',
7: '币种',
8: '产品名称',
9: '产品编号',
10: '产品型号',
11: '原价',
12: '折扣率',
13: '单价',
14: '数量',
15: '产品描述',
16: 'AVOIR',
})
sheet.append(record.headers)
categories = dict()
clients = dict()
try:
for i, item in enumerate(data, 1):
flow.react(Wait)
number: str = item['number']
logger.info('[%d/%d] Preprocessing data for %s', i, len(data), number)
progress['number'] = number
progress['index'] = i-1
if (category := categories.get(o := item['category_id'])) is None:
if 'error' in (category := fetch(f'{base}/categories/{o}.json?api_token={profile.token}')):
error = dict.get(category, 'error')
response = dict.get(category, 'status')
logger.warning("Error while fetching field 'category' (status: %s, message: %s); skipping", response, error)
continue
if (client := clients.get(o := item['client_id'])) is None:
if 'error' in (client := fetch(f'{base}/clients/{o}.json?api_token={profile.token}')):
error = dict.get(category, 'error')
response = dict.get(category, 'status')
logger.warning("Error while fetching field 'client' (status: %s, message: %s); skipping", response, error)
continue
identity = client['shortcut'] if client['company'] else profile.person
date: str = item['issue_date']
kind: str = item['kind']
total = float(item['price_net'])
positions: list = item['positions']
for position in positions:
code: str = position['code']
product: str = position['name']
description: str = position['description']
price = float(position['price_net'] or '0')
discount = float(position['discount_percent'] or '0')
quantity = float(position['quantity'] or '0')
record.clear()
record[1] = profile.format(number)
record[2] = number.replace('/', '-')
record[3] = date
record[4] = category['name']
record[5] = profile.name
record[6] = identity
record[7] = 'USD'
match kind:
case 'vat':
record[8] = product
record[9] = (p.at('产品型号', code) or p.at('产品名称', product)).get('产品编号')
record[10] = code
record[11] = '%.2f' % price
record[12] = '%g%%' % discount
record[13] = '%.2f' % (price * (1 - discount / 100))
record[14] = '%g' % quantity
record[15] = description
case 'correction':
record[8] = p.at('产品编号', profile.remise).get('产品名称')
record[9] = profile.remise
record[13] = '0'
record[14] = '0'
record[16] = '%.2f' % total
positions.clear()
if record[9] is None: logger.warning("Could not identify product '%s'", product)
sheet.append(record.data)
except Skip:
pass
except Cancel:
continue
except Exception as e:
logger.error('Error while processing data', exc_info=e)
status = Status.STANDBY
continue
if sheet.max_row < 2:
logger.warning("No data; skipping")
continue
try:
filename = f'Order-Import-{profile.name}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}.xlsx'
file = Path(parameters['directory']).joinpath(filename)
logger.info('Saving document at %s', str(file))
flow.react(Wait)
workbook.save(file)
except Skip:
pass
except Cancel:
continue
except Exception as e:
logger.error('Error while saving document', exc_info=e)
status = Status.STANDBY
continue
try:
logger.info('Uploading data...')
progress.clear()
progress['task'] = 'Task 3 of 4'
t2.clear()
t2.start()
driver.switch_to.new_window('tab')
driver.get(WEBURL % 'order/importOrder')
click(".product-import-img-box .import-img-radio:nth-child(2) .mm-radio-group > label:nth-child(2) .mm-radio-input", condition=None)
click(".product-import-img-box .mm-selector-rendered")
click(".mm-outside.mm-select-dropdown ul li:nth-child(%d) span" % (1 if options.get('draft') else 6))
locate(".big-file-upload input", wait=False).send_keys(str(file))
click(".product-import-img-footer button")
logger.info('Done')
except Skip:
pass
except Cancel:
continue
except Exception as e:
logger.error('Error while uploading document', exc_info=e)
status = Status.STANDBY
continue
while not flow.react(Wait, Sleep):
try:
err = locate(".mm-tbody table tbody tr:nth-child(1) td:nth-child(5) .okki-space-item:nth-child(1) button", wait=False)
break
except: pass
try: click(".product-import-img-footer button.mm-button__primary", wait=False)
except: pass
try: click(".list-header-wrap button", wait=False)
except: pass
if err.get_attribute('disabled') is None:
click(err, condition=None)
logger.warning('Incomplete import detected; downloading 1 related document')
flow.react(Sleep)
try: click(".mm-tbody table tbody tr:nth-child(1) td:nth-child(3) a")
except: continue
flow.react(Sleep)
class Parse:
def __init__(self, url: str):
from urllib.parse import urlsplit, parse_qs
parts = list(urlsplit(url))
query = parse_qs(parts[3])
self.parts = parts
self.query = json.loads(query['query'][0]) if 'query' in query else dict()
def encode(self, extra=None):
from urllib.parse import urlencode, urlunsplit
query = { 'query': json.dumps(self.query, separators=(',', ':')) }
if extra is not None: query.update(extra)
self.parts[3] = urlencode(query)
return urlunsplit(self.parts)
def get(self, key: str):
return self.query[key]
def set(self, key: str, value):
self.query[key] = value
progress['task'] = 'Task 4 of 4'
progress['limit'] = len(data)
t2.clear()
t2.start()
index = 0
attempts = 0
while index < len(data):
try:
attempts += 1
if len(driver.window_handles) > 3: driver.close()
driver.switch_to.window(driver.window_handles[2])
item = data[index]
kind = item['kind']
title = re.search(r'O\d+', item['title'] or '')
number = profile.format(item['number'])
positions = item['positions']
opportunity = None
if kind != 'vat':
logger.info('[%d/%d] Undesired invoice type; skipping %s', index+1, len(data), number)
raise Skip()
if attempts > parameters['attempts']:
logger.warning('Exhausted all allowed attempts; skipping %s', number)
raise Skip()
progress['number'] = number
progress['index'] = index
flow.allow(Skip, Cancel)
flow.react(Wait)
try:
for page in count(1):
url = Parse(driver.current_url)
url.set('page_size', 1)
url.set('page', page)
url.set('query_filters', [{
'field': 'order_no',
'name': '订单号',
'operator': 'match',
'value': number,
'object_name': 'objOrder',
'field_type': 35,
'unit': '',
'array_flag': 0
}])
driver.get(url.encode())
flow.react(Wait, Sleep)
url = Parse(driver.current_url)
if url.get('page') != page: raise Exception(number)
flow.react(Sleep)
link = locate(".virtual-list-wrap .vue-recycle-scroller .vue-recycle-scroller__item-wrapper > div:nth-child(1) .cell[data-cci='1'] a", wait=False)
if link.text != number: continue
logger.info('[%d/%d] Processing %s...', index+1, len(data), number)
link.click()
driver.switch_to.window(driver.window_handles[3])
flow.react(Wait)
click(".sticky .okki-space-item:nth-child(1) button")
break
except NoSuchElementException:
logger.warning("Could not find invoice '%s'; skipping", number)
raise Skip()
except Action as e:
raise e
except Exception as e:
logger.error("Error while looking up invoice '%s'", number, exc_info=e)
status = Status.STANDBY
continue
if title is not None and (match := title[0]):
try:
base = WEBURL % 'crm/business/list'
driver.switch_to.new_window('tab')
url = Parse(base)
driver.get(url.encode({ 'mode': 'list' }))
flow.react(Wait, Sleep)
try: click(".new-wrapper .paas-next-invoice-list-filter-line-wrapper .okki-btn-background-ghost", wait=False)
except: pass
for page in count(1):
url = Parse(base)
url.set('keyword', match)
url.set('search_field', 'serial_keyword')
url.set('curPage', page)
url.set('pageSize', 1)
driver.get(url.encode({ 'keyword': match, 'search_field': 'serial_keyword' }))
flow.react(Wait, Sleep)
url = Parse(driver.current_url)
if url.get('curPage') != page: raise Exception(match)
flow.react(Sleep)
cell = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(3) .ow-serial-read-pretty_ellipsis", wait=False)
if cell.text != match: continue
link = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(6) a", wait=False)
opportunity = link.text
break
except NoSuchElementException:
logger.warning("Could not find opportunity '%s'", match)
except Action as e:
raise e
except Exception as e:
logger.error("Error while looking up opportunity '%s'", match, exc_info=e)
status = Status.STANDBY
continue
finally:
driver.close()
driver.switch_to.window(driver.window_handles[3])
if opportunity is not None:
try:
flow.react(Wait)
label = locate("label.paas-form-item-label")
dropdown = locate("label.paas-form-item-label[title='商机'] + div input")
for iteration in count(1):
dropdown.click()
dropdown.clear()
dropdown.send_keys(opportunity)
menu = locate(".okki-select-dropdown")
menuitems = menu.find_elements(By.CSS_SELECTOR, ".rc-virtual-list-holder-inner > div")
try:
for menuitem in menuitems:
if menuitem.get_attribute('label') == opportunity:
click(menuitem)
raise StopIteration()
except StopIteration: break
except: pass
assert iteration < parameters['attempts'], "Exceeded maximum retry attempts"
click(label, condition=None)
except Action as e:
raise e
except Exception as e:
logger.warning("Could not select opportunity '%s'", opportunity, exc_info=e)
try:
pagination = 10
click(".okki-pagination-options-size-changer")
click(".okki-select-dropdown .rc-virtual-list-holder-inner > div:nth-child(1)")
except:
logger.warning('Unable to setup pagination; this may cause issues')
try:
ids = list()
wrapper = locate(".paas-order-product-list .row-items", condition=None)
for page in count(1):
hits = 0
iteration = 0
flow.react(Wait)
while hits < pagination and iteration < parameters['attempts']:
iteration += 1
height = int(wrapper.get_attribute('clientHeight')) if iteration > 1 else 0
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper)
driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height)
rows = wrapper.find_elements(By.CSS_SELECTOR, ".row-item")
for row in reversed(rows) if iteration > 1 else rows:
flow.react(Wait)
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper)
driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height)
serial = row.text.split('\n', 1)[0].strip()
if not serial or serial in ids: continue
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row)
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper)
value = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='4'] input").get_attribute('value')
driver.execute_script("arguments[0].scroll(400, 0);", wrapper)
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row)
driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper)
flow.react(Sleep)
target = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='6'] input")
if (target.get_attribute('value') == '0'):
target.send_keys(Keys.BACKSPACE)
target.send_keys(value)
ids.append(serial)
hits += 1
if len(ids) >= len(positions): break
button = locate(".text-right li.okki-pagination-next button", condition=None)
if button.get_attribute('disabled') is not None and len(ids) < len(positions):
raise Exception('Product list imcomplete; expected %d, got %d' % (len(positions), len(ids)))
flow.react(Wait)
click(button)
except Action as e:
raise e
except Exception as e:
logger.error('Error while modifying invoice', exc_info=e)
status = Status.STANDBY
continue
try:
click(".ow-box button.okki-btn-round", wait=False)
flow.react(Sleep)
except Exception as e:
logger.warning('Unable to unset additional fees', exc_info=e)
flow.react(Wait)
flow.deter(Skip, Cancel)
click(".sticky.bottom-0 button.okki-btn-primary", condition=None)
flow.react(Sleep)
driver.close()
except Skip:
pass
except Cancel:
break
except Exception as e:
logger.error('Unexpected error', exc_info=e)
status = Status.STANDBY
continue
index += 1
attempts = 0
if __name__ == '__main__':
try:
logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M")
logger = logging.getLogger()
level = logging.getLevelNamesMapping().get(args.log_level, 'INFO')
logger.setLevel(level)
opts = ChromeOptions()
opts.enable_downloads = True
opts.add_argument('--deny-permission-prompts')
opts.add_experimental_option('prefs', { 'download.default_directory': args.directory })
with keep.presenting():
driver = Chrome(options=opts)
status = main(driver)
except KeyboardInterrupt:
status = 0
except Exception as e:
logger.critical('Fatal error', exc_info=e)
status = 1
finally:
driver.quit()
exit(status)