import argparse import openpyxl import logging import json import re from selenium.webdriver import Chrome, ChromeOptions from selenium.webdriver.remote.webdriver import WebDriver from selenium.common.exceptions import TimeoutException, NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from common.utils import * from common.timer import Timer from common.jsonrpc2 import ServiceProvider from common.actionflow import Action, ActionFlow from enum import Enum from wakepy import keep from pathlib import Path from datetime import datetime from itertools import count from urllib3 import PoolManager parser = argparse.ArgumentParser(description="Order Import") parser.add_argument('account', type=str, nargs='?') parser.add_argument('password', type=str, nargs='?') parser.add_argument('-d', '--directory', type=str, default=str(Path.home().joinpath('Downloads'))) parser.add_argument('-o', '--profile', nargs='+', action='append', required=True) parser.add_argument('-t', '--timeout', type=int, default=60) parser.add_argument('-r', '--attempts', type=int, default=3) parser.add_argument('-i', '--interval', type=int, default=3) parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']) args = parser.parse_args() WEBURL = "https://crm.xiaoman.cn/%s" APIURL = "https://%s.vosfactures.fr" def main(driver: WebDriver, logger = logging.getLogger('main')): parameters = vars(args) http = PoolManager() sp = ServiceProvider.default() class Status(Enum): BUSY = 0 READY = 1 RUNNING = 2 STANDBY = 3 class Profile: def __init__(self, name, subdomain, token, remise, person=None, prefix=None, suffix=None): self.name = name self.subdomain = subdomain self.token = token self.remise = remise self.person = person self.prefix = prefix self.suffix = suffix def format(self, number: str): result = ''.join(filter(bool, [self.prefix, number, self.suffix])) return result t1 = Timer() t2 = Timer() options = dict() status = Status.BUSY def begin(opts: dict, args: dict): nonlocal status options.update(opts) status = Status.RUNNING parameters.update(args) t1.clear() t1.start() def pause(): nonlocal status status = Status.STANDBY t1.pause() t2.pause() def resume(): nonlocal status status = Status.RUNNING driver.switch_to.window(driver.current_window_handle) t1.start() t2.start() sp.add(begin, pause, resume) sp.set('status', lambda: status.name) sp.set('uptime', lambda: [t1.delta()]) try: profiles = [ Profile(**{ k.lower().strip(): v.strip() for k, v in map(lambda o: str.split(o, '=', 2), p) }) for p in parameters['profile'] ] except Exception as e: logger.critical('Unable to load profiles', exc_info=e) return 2 try: sp.set('context', lambda: { 'profiles': list(map(vars, profiles)), 'parameters': parameters }) driver.get(sp.run()) except Exception as e: logger.critical('Unable to load starup page', exc_info=e) return 3 try: driver.switch_to.new_window('tab') driver.set_page_load_timeout(parameters['timeout']) driver.get(WEBURL % 'product') except TimeoutException: logger.warning('Timeout') driver.execute_script("window.stop();") setup(driver, parameters) until(lambda x: 'loginProgress' in x.find_element(By.TAG_NAME, "body").get_attribute('class'), watch=False) logger.info('Waiting for authentication...') if (account := parameters['account']) and (password := parameters['password']): try: logger.info('Logging in as %s (%s)', str.split(account, '@', 1).pop(0).capitalize(), account) locate("input.account").send_keys(account) locate("input#password").send_keys(password) click("input.agree-checkbox") click("button.login-btn") except Exception as e: logger.critical('Unable to login to %s', account, exc_info=e) return 3 while True: try: locate("#container", wait=False) logger.info('Done') break except: sleep(parameters['interval']) class ProductInfo: def __init__(self, file): self.wb = openpyxl.load_workbook(file, read_only=True) self.indices = dict() self.headers = dict() def index(self, *keys: str): self.indices.clear() self.headers.clear() for col in range(1, self.wb.active.max_column + 1): value = str(self.wb.active.cell(1, col).value) self.headers[value] = col for row in range(1, self.wb.active.max_row + 1): for key, col in map(lambda k: (k, self.headers[k]), keys): value = str(self.wb.active.cell(row, col).value) self.indices.setdefault(key, dict())[value] = row def at(self, axis: str, index) -> dict: if (a := self.indices.get(axis)) is None or (b := dict.get(a, index)) is None: return dict() return { k: self.wb.active.cell(b, c).value for k, c in self.headers.items() } try: logger.info('Downloading product list...') click("header .okki-space .okki-space-item:nth-child(1) button") click(".okki-dropdown button") sleep(parameters['interval']) click(".okki-modal.product-export-wrap .mm-selector-rendered") click(".mm-outside.ui-field-selector-popper .selector-area:nth-child(1) button") click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary") while True: try: click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary") sleep(parameters['interval']) click(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) button.okki-btn-link", wait=False) filename = locate(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) > div > div:nth-child(1) span").get_attribute('title') break except: sleep(parameters['interval']) file = Path(parameters['directory']).joinpath(filename) until(lambda _: file.exists(), watch=False) p = ProductInfo(file) driver.close() driver.switch_to.window(driver.window_handles[0]) logger.info('Indexing product information...') p.index('产品编号', '产品名称', '产品型号') logger.info('Done') status = Status.READY except Exception as e: logger.critical('Unable to load products', exc_info=e) return 4 def fetch(url: str, method = 'GET', retry = parameters['attempts']): for attempt in range(1, retry + 1): try: response = http.request(method, url) result = response.json() return result except Exception as e: logger.warning('Error while fetching data from %s, retrying... (%d)', url, attempt, exc_info=e) assert attempt < retry, "Exceeded maximum retry attempts" class Wait(Action): @classmethod def prepare(cls): return True @classmethod def perform(cls): if status == Status.RUNNING: return False sleep(0.2); return True class Sleep(Action): @classmethod def prepare(cls): return True @classmethod def perform(cls): sleep(parameters['interval']) return False class Cancel(Action): @classmethod def prepare(cls): nonlocal status status = Status.RUNNING return True @classmethod def perform(cls): nonlocal status status = Status.READY driver.switch_to.window(driver.window_handles[0]) raise cls class Skip(Action): @classmethod def prepare(cls): return True @classmethod def perform(cls): driver.switch_to.window(driver.current_window_handle) raise cls flow = ActionFlow() flow.stage(Wait, Sleep, Cancel, Skip) profile = None progress = { 'task': '' } selection = 0 sp.add(pairs=[ (k.lower(), v) for k, v in flow ]) sp.set('actions', lambda: flow.capabilities()) sp.set('progress', lambda: progress) while not flow.react(Wait): try: for i in range(len(driver.window_handles), 1, -1): driver.switch_to.window(driver.window_handles[i-1]) driver.close() driver.switch_to.window(driver.window_handles[i-2]) if options.get('all'): profile = profiles[selection] selection += 1 else: name = options.pop('profile') profile = next(filter(lambda o: o.name == name, profiles)) except (IndexError, KeyError): logger.info('Done') status = Status.READY continue except StopIteration: logger.error("Invalid profile '%s'", name) status = Status.STANDBY continue except Exception as e: logger.error('Unexpected error', exc_info=e) status = Status.STANDBY continue progress.clear() progress['task'] = 'Task 1 of 4' t2.clear() t2.start() sp.pop('uptime') sp.set('uptime', lambda: [t1.delta(), t2.delta()]) base = APIURL % profile.subdomain data = list() df = options.get('datefrom') dt = options.get('dateto') types = ['vat'] if options.get('avoir'): types.append('correction') logger.info('Profile selected: %s', profile.name) logger.info('Date from %s to %s', df, dt) flow.allow(Cancel) flow.deter(Skip) for page in count(1): try: result = fetch(f'{base}/invoices.json?{'&'.join([f'kinds%5B%5D={k}' for k in types])}&api_token={profile.token}&include_positions=true&per_page=25&page={page}&period=more&date_from={df}&date_to={dt}') if 'message' in result: raise Exception(result['message']) if not isinstance(result, list): raise TypeError() if len(result) == 0: break logger.info('Downloading invoices (%d)', page) data.extend(result) flow.react(Wait, Sleep) except Skip: pass except Cancel: break except Exception as e: logger.error('Error while fetching data from %s', base, exc_info=e) break if len(data) == 0: logger.warning('Server returned an empty response') continue logger.info('Initializing Workbook...') progress['task'] = 'Task 2 of 4' progress['limit'] = len(data) progress['index'] = 0 t2.clear() t2.start() workbook = openpyxl.Workbook() sheet = workbook.active class Record: def __init__(self, fields: dict[int, str]): self.headers = fields self.data = dict() def clear(self): self.data.clear() def __setitem__(self, key, value): if key not in self.headers: raise KeyError(key) self.data[key] = value def __getitem__(self, key): if key not in self.headers: raise KeyError(key) return self.data[key] # Required fields # See record = Record({ 1: '订单号', 2: '订单名称', 3: '订单日期', 4: '当前处理人', 5: '业绩归属部门', 6: '客户编号', 7: '币种', 8: '产品名称', 9: '产品编号', 10: '产品型号', 11: '原价', 12: '折扣率', 13: '单价', 14: '数量', 15: '产品描述', 16: 'AVOIR', }) sheet.append(record.headers) categories = dict() clients = dict() try: for i, item in enumerate(data, 1): flow.react(Wait) number: str = item['number'] logger.info('[%d/%d] Preprocessing data for %s', i, len(data), number) progress['number'] = number progress['index'] = i-1 if (category := categories.get(o := item['category_id'])) is None: if 'error' in (category := fetch(f'{base}/categories/{o}.json?api_token={profile.token}')): error = dict.get(category, 'error') response = dict.get(category, 'status') logger.warning("Error while fetching field 'category' (status: %s, message: %s); skipping", response, error) continue if (client := clients.get(o := item['client_id'])) is None: if 'error' in (client := fetch(f'{base}/clients/{o}.json?api_token={profile.token}')): error = dict.get(category, 'error') response = dict.get(category, 'status') logger.warning("Error while fetching field 'client' (status: %s, message: %s); skipping", response, error) continue identity = client['shortcut'] if client['company'] else profile.person date: str = item['issue_date'] kind: str = item['kind'] total = float(item['price_net']) positions: list = item['positions'] for position in positions: code: str = position['code'] product: str = position['name'] description: str = position['description'] price = float(position['price_net'] or '0') discount = float(position['discount_percent'] or '0') quantity = float(position['quantity'] or '0') record.clear() record[1] = profile.format(number) record[2] = number.replace('/', '-') record[3] = date record[4] = category['name'] record[5] = profile.name record[6] = identity record[7] = 'USD' match kind: case 'vat': record[8] = product record[9] = (p.at('产品型号', code) or p.at('产品名称', product)).get('产品编号') record[10] = code record[11] = '%.2f' % price record[12] = '%g%%' % discount record[13] = '%.2f' % (price * (1 - discount / 100)) record[14] = '%g' % quantity record[15] = description case 'correction': record[8] = p.at('产品编号', profile.remise).get('产品名称') record[9] = profile.remise record[13] = '0' record[14] = '0' record[16] = '%.2f' % total positions.clear() if record[9] is None: logger.warning("Could not identify product '%s'", product) sheet.append(record.data) except Skip: pass except Cancel: continue except Exception as e: logger.error('Error while processing data', exc_info=e) status = Status.STANDBY continue if sheet.max_row < 2: logger.warning("No data; skipping") continue try: filename = f'Order-Import-{profile.name}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}.xlsx' file = Path(parameters['directory']).joinpath(filename) logger.info('Saving document at %s', str(file)) flow.react(Wait) workbook.save(file) except Skip: pass except Cancel: continue except Exception as e: logger.error('Error while saving document', exc_info=e) status = Status.STANDBY continue try: logger.info('Uploading data...') progress.clear() progress['task'] = 'Task 3 of 4' t2.clear() t2.start() driver.switch_to.new_window('tab') driver.get(WEBURL % 'order/importOrder') click(".product-import-img-box .import-img-radio:nth-child(2) .mm-radio-group > label:nth-child(2) .mm-radio-input", condition=None) click(".product-import-img-box .mm-selector-rendered") click(".mm-outside.mm-select-dropdown ul li:nth-child(%d) span" % (1 if options.get('draft') else 6)) locate(".big-file-upload input", wait=False).send_keys(str(file)) click(".product-import-img-footer button") logger.info('Done') except Skip: pass except Cancel: continue except Exception as e: logger.error('Error while uploading document', exc_info=e) status = Status.STANDBY continue while not flow.react(Wait, Sleep): try: err = locate(".mm-tbody table tbody tr:nth-child(1) td:nth-child(5) .okki-space-item:nth-child(1) button", wait=False) break except: pass try: click(".product-import-img-footer button.mm-button__primary", wait=False) except: pass try: click(".list-header-wrap button", wait=False) except: pass if err.get_attribute('disabled') is None: click(err, condition=None) logger.warning('Incomplete import detected; downloading 1 related document') flow.react(Sleep) try: click(".mm-tbody table tbody tr:nth-child(1) td:nth-child(3) a", wait=False) except: continue flow.react(Sleep) class Parse: def __init__(self, url: str): from urllib.parse import urlsplit, parse_qs parts = list(urlsplit(url)) query = parse_qs(parts[3]) self.parts = parts self.query = json.loads(query['query'][0]) if 'query' in query else dict() def encode(self, extra=None): from urllib.parse import urlencode, urlunsplit query = { 'query': json.dumps(self.query, separators=(',', ':')) } if extra is not None: query.update(extra) self.parts[3] = urlencode(query) return urlunsplit(self.parts) def get(self, key: str): return self.query[key] def set(self, key: str, value): self.query[key] = value progress['task'] = 'Task 4 of 4' progress['limit'] = len(data) t2.clear() t2.start() index = 0 attempts = 0 while index < len(data): try: attempts += 1 if len(driver.window_handles) > 3: driver.close() driver.switch_to.window(driver.window_handles[2]) item = data[index] kind = item['kind'] title = re.search(r'O\d+', item['title']) number = profile.format(item['number']) positions = item['positions'] opportunity = None if kind != 'vat': logger.info('[%d/%d] Undesired invoice type; skipping %s', index+1, len(data), number) raise Skip() if attempts > parameters['attempts']: logger.warning('Exhausted all allowed attempts; skipping %s', number) raise Skip() progress['number'] = number progress['index'] = index flow.allow(Skip, Cancel) flow.react(Wait) try: for page in count(1): url = Parse(driver.current_url) url.set('page_size', 1) url.set('page', page) url.set('query_filters', [{ 'field': 'order_no', 'name': '订单号', 'operator': 'match', 'value': number, 'object_name': 'objOrder', 'field_type': 35, 'unit': '', 'array_flag': 0 }]) driver.get(url.encode()) flow.react(Wait, Sleep) url = Parse(driver.current_url) if url.get('page') != page: raise Exception(number) flow.react(Sleep) link = locate(".virtual-list-wrap .vue-recycle-scroller .vue-recycle-scroller__item-wrapper > div:nth-child(1) .cell[data-cci='1'] a", wait=False) if link.text != number: continue logger.info('[%d/%d] Processing %s...', index+1, len(data), number) link.click() driver.switch_to.window(driver.window_handles[3]) flow.react(Wait) click(".sticky .okki-space-item:nth-child(1) button") break except NoSuchElementException: logger.warning("Could not find invoice '%s'; skipping", number) raise Skip() except Action as e: raise e except Exception as e: logger.error("Error while looking up invoice '%s'", number, exc_info=e) status = Status.STANDBY continue if title is not None and (match := title[0]): try: base = WEBURL % 'crm/business/list' driver.switch_to.new_window('tab') url = Parse(base) driver.get(url.encode({ 'mode': 'list' })) flow.react(Wait, Sleep) try: click(".new-wrapper .paas-next-invoice-list-filter-line-wrapper .okki-btn-background-ghost", wait=False) except: pass for page in count(1): url = Parse(base) url.set('keyword', match) url.set('search_field', 'serial_keyword') url.set('curPage', page) url.set('pageSize', 1) driver.get(url.encode({ 'keyword': match, 'search_field': 'serial_keyword' })) flow.react(Wait, Sleep) url = Parse(driver.current_url) if url.get('curPage') != page: raise Exception(match) flow.react(Sleep) cell = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(3) .ow-serial-read-pretty_ellipsis", wait=False) if cell.text != match: continue link = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(6) a", wait=False) opportunity = link.text break except NoSuchElementException: logger.warning("Could not find opportunity '%s'", match) except Action as e: raise e except Exception as e: logger.error("Error while looking up opportunity '%s'", match, exc_info=e) status = Status.STANDBY continue finally: driver.close() driver.switch_to.window(driver.window_handles[3]) if opportunity is not None: try: flow.react(Wait) label = locate("label.paas-form-item-label") dropdown = locate("label.paas-form-item-label[title='商机'] + div input") for iteration in count(1): dropdown.click() dropdown.clear() dropdown.send_keys(opportunity) menu = locate(".okki-select-dropdown") menuitems = menu.find_elements(By.CSS_SELECTOR, ".rc-virtual-list-holder-inner > div") try: for menuitem in menuitems: if menuitem.get_attribute('label') == opportunity: click(menuitem) raise StopIteration() except StopIteration: break except: pass assert iteration < parameters['attempts'], "Exceeded maximum retry attempts" click(label, condition=None) except Action as e: raise e except Exception as e: logger.warning("Could not select opportunity '%s'", opportunity, exc_info=e) try: pagination = 10 click(".okki-pagination-options-size-changer") click(".okki-select-dropdown .rc-virtual-list-holder-inner > div:nth-child(1)") except: logger.warning('Unable to setup pagination; this may cause issues') try: ids = list() wrapper = locate(".paas-order-product-list .row-items", condition=None) for page in count(1): hits = 0 iteration = 0 flow.react(Wait) while hits < pagination and iteration < parameters['attempts']: iteration += 1 height = int(wrapper.get_attribute('clientHeight')) if iteration > 1 else 0 driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height) rows = wrapper.find_elements(By.CSS_SELECTOR, ".row-item") for row in reversed(rows) if iteration > 1 else rows: flow.react(Wait) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height) serial = row.text.split('\n', 1)[0].strip() if not serial or serial in ids: continue driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) value = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='4'] input").get_attribute('value') driver.execute_script("arguments[0].scroll(400, 0);", wrapper) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) flow.react(Sleep) target = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='6'] input") if (target.get_attribute('value') == '0'): target.send_keys(Keys.BACKSPACE) target.send_keys(value) ids.append(serial) hits += 1 if len(ids) >= len(positions): break button = locate(".text-right li.okki-pagination-next button", condition=None) if button.get_attribute('disabled') is not None and len(ids) < len(positions): raise Exception('Product list imcomplete; expected %d, got %d' % (len(positions), len(ids))) flow.react(Wait) click(button) except Action as e: raise e except Exception as e: logger.error('Error while modifying invoice', exc_info=e) status = Status.STANDBY continue try: click(".ow-box button.okki-btn-round", wait=False) flow.react(Sleep) except Exception as e: logger.warning('Unable to unset additional fees', exc_info=e) flow.react(Wait) flow.deter(Skip, Cancel) click(".sticky.bottom-0 button.okki-btn-primary", condition=None) flow.react(Sleep) driver.close() except Skip: pass except Cancel: break except Exception as e: logger.error('Unexpected error', exc_info=e) status = Status.STANDBY continue index += 1 attempts = 0 if __name__ == '__main__': try: logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M") logger = logging.getLogger() level = logging.getLevelNamesMapping().get(args.log_level, 'INFO') logger.setLevel(level) opts = ChromeOptions() opts.enable_downloads = True opts.add_argument('--deny-permission-prompts') opts.add_experimental_option('prefs', { 'download.default_directory': args.directory }) with keep.presenting(): driver = Chrome(options=opts) status = main(driver) except KeyboardInterrupt: status = 0 except Exception as e: logger.critical('Fatal error', exc_info=e) status = 1 finally: driver.quit() exit(status)