import unicodedata import argparse import openpyxl import logging import json import re from selenium.webdriver import Chrome, ChromeOptions from selenium.webdriver.remote.webdriver import WebDriver from selenium.common.exceptions import TimeoutException, NoSuchElementException from selenium.webdriver.common.by import By from selenium.webdriver.common.keys import Keys from common.utils import * from common.timer import Timer from common.jsonrpc2 import ServiceProvider from common.actionflow import Action, ActionFlow from enum import Enum from wakepy import keep from pathlib import Path from datetime import datetime from itertools import repeat, count from urllib3 import PoolManager parser = argparse.ArgumentParser(description="Order Import") parser.add_argument('account', type=str, nargs='?') parser.add_argument('password', type=str, nargs='?') parser.add_argument('-d', '--directory', type=str, default=str(Path.home().joinpath('Downloads'))) parser.add_argument('-o', '--profile', nargs='+', action='append', required=True) parser.add_argument('-t', '--timeout', type=int, default=60) parser.add_argument('-r', '--attempts', type=int, default=3) parser.add_argument('-i', '--interval', type=int, default=3) parser.add_argument('-l', '--log-level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']) args = parser.parse_args() WEBURL = "https://crm.xiaoman.cn/%s" APIURL = "https://%s.vosfactures.fr" def main(driver: WebDriver, logger = logging.getLogger('main')): parameters = vars(args) http = PoolManager() sp = ServiceProvider.default() class Status(Enum): IDLE = 0 READY = 1 RUNNING = 3 STANDBY = 4 class Profile: def __init__(self, name, subdomain, token, remise, person=None, prefix=None, suffix=None): self.name = name self.subdomain = subdomain self.token = token self.remise = remise self.person = person self.prefix = prefix self.suffix = suffix def format(self, number: str): result = ''.join(filter(bool, [self.prefix, number, self.suffix])) return result t1 = Timer() t2 = Timer() options = dict() status = Status.IDLE def begin(opts: dict, args: dict): nonlocal status options.update(opts) status = Status.RUNNING parameters.update(args) t1.clear() t1.start() def pause(): nonlocal status status = Status.STANDBY t1.pause() t2.pause() def resume(): nonlocal status status = Status.RUNNING driver.switch_to.window(driver.current_window_handle) t1.start() t2.start() sp.set('begin', begin) sp.set('pause', pause) sp.set('resume', resume) sp.set('status', lambda: status.name) sp.set('uptime', lambda: [t1.delta()]) try: profiles = [ Profile(**{ k.lower().strip(): v.strip() for k, v in map(lambda o: str.split(o, '=', 2), p) }) for p in parameters['profile'] ] except Exception as e: logger.critical('Unable to load profiles', exc_info=e) return 2 try: sp.set('context', lambda: { 'profiles': list(map(vars, profiles)), 'parameters': parameters }) driver.get(sp.run()) except Exception as e: logger.critical('Unable to load starup page', exc_info=e) return 3 try: driver.switch_to.new_window('tab') driver.set_page_load_timeout(parameters['timeout']) driver.get(WEBURL % 'product') except TimeoutException: logger.warning('Timeout') driver.execute_script("window.stop();") setup(driver, parameters) until(lambda x: 'loginProgress' in x.find_element(By.TAG_NAME, "body").get_attribute('class'), watch=False) logger.info('Waiting for authentication to complete...') if (account := parameters['account']) and (password := parameters['password']): try: logger.info('Logging in as %s (%s)', str.split(account, '@', 1).pop(0).capitalize(), account) locate("input.account").send_keys(account) locate("input#password").send_keys(password) click("input.agree-checkbox") click("button.login-btn") except Exception as e: logger.critical('Unable to login to %s', account, exc_info=e) return 3 while True: try: locate("#container", wait=False) logger.info('Done') break except: sleep(parameters['interval']) class ProductInfo: def __init__(self, file): self.wb = openpyxl.load_workbook(file, read_only=True) self.headers = list() for col in range(1, self.wb.active.max_column + 1): value = str(self.wb.active.cell(1, col).value) self.headers.append(value) def search(self, by: str, needle) -> dict: a = self.headers.index(by) b = None for row in range(1, self.wb.active.max_row + 1): if self.wb.active.cell(row, a+1).value == needle: b = row break if b is None: return dict() return { k: self.wb.active.cell(b, c+1).value for c, k in enumerate(self.headers) } try: logger.info('Downloading product list...') click("header .okki-space .okki-space-item:nth-child(1) button") click(".okki-dropdown button") sleep(parameters['interval']) click(".okki-modal.product-export-wrap .mm-selector-rendered") click(".mm-outside.ui-field-selector-popper .selector-area:nth-child(1) button") click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary") while True: try: click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary") sleep(parameters['interval']) click(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) button.okki-btn-link", wait=False) filename = locate(".okki-modal.product-export-wrap .okki-modal-body .virtual-list-wrap .vue-recycle-scroller__item-wrapper > div:nth-child(1) > div > div:nth-child(1) span").get_attribute('title') break except: sleep(parameters['interval']) file = Path(parameters['directory']).joinpath(filename) until(lambda _: file.exists(), watch=False) p = ProductInfo(file) driver.close() driver.switch_to.window(driver.window_handles[0]) status = Status.READY logger.info('Done') except Exception as e: logger.critical('Unable to load products', exc_info=e) return 4 def fetch(url: str, method = 'GET', retry = parameters['attempts']): for attempt in range(1, retry + 1): try: response = http.request(method, url) result = response.json() return result except Exception as e: logger.warning('Error while fetching data from %s, retrying... (%d)', url, attempt, exc_info=e) assert attempt < retry class Wait(Action): @classmethod def prepare(cls): return True @classmethod def perform(cls): if status == Status.RUNNING: return False sleep(0.2); return True class Sleep(Action): @classmethod def prepare(cls): return True @classmethod def perform(cls): sleep(parameters['interval']) return False class Cancel(Action): @classmethod def prepare(cls): nonlocal status status = Status.RUNNING return True @classmethod def perform(cls): nonlocal status status = Status.READY driver.switch_to.window(driver.window_handles[0]) raise cls class Skip(Action): @classmethod def prepare(cls): return True @classmethod def perform(cls): driver.switch_to.window(driver.current_window_handle) raise cls flow = ActionFlow() flow.append(Wait) flow.append(Sleep) flow.append(Cancel) flow.append(Skip) profile = None progress = { 'task': '' } selection = 0 sp.set('actions', lambda: flow.capabilities()) sp.set('cancel', lambda: flow.do(Cancel, force=True)) sp.set('skip', lambda: flow.do(Skip, force=True)) sp.set('progress', lambda: progress) while True: try: flow.allow(Wait) flow.allow(Sleep) flow.do(Wait) flow.react() for i in range(len(driver.window_handles), 1, -1): driver.switch_to.window(driver.window_handles[i-1]) driver.close() driver.switch_to.window(driver.window_handles[i-2]) if options.get('all'): profile = profiles[selection] selection += 1 else: name = options.pop('profile') profile = next(filter(lambda o: o.name == name, profiles)) except (IndexError, KeyError): logger.info('Done') status = Status.READY continue except StopIteration: logger.error("Invalid profile '%s'", name) status = Status.STANDBY continue except Exception as e: logger.error('Unexpected error', exc_info=e) status = Status.STANDBY continue progress.clear() progress['task'] = 'Task 1 of 4' t2.clear() t2.start() sp.pop('uptime') sp.set('uptime', lambda: [t1.delta(), t2.delta()]) base = APIURL % profile.subdomain data = list() df = options.get('datefrom') dt = options.get('dateto') types = ['vat'] if options.get('avoir'): types.append('correction') logger.info('Profile selected: %s', profile.name) logger.info('Date from %s to %s', df, dt) flow.allow(Cancel) flow.allow(Skip, False) for page in count(1): try: result = fetch(f'{base}/invoices.json?{'&'.join([f'kinds%5B%5D={k}' for k in types])}&api_token={profile.token}&include_positions=true&per_page=25&page={page}&period=more&date_from={df}&date_to={dt}') if 'message' in result: raise Exception(result['message']) if not isinstance(result, list): raise TypeError() if len(result) == 0: break logger.info('Downloading invoices (%d)', page) data.extend(result) flow.do(Wait) flow.do(Sleep) flow.react() except Skip: pass except Cancel: break except Exception as e: logger.error('Error while fetching data from %s', base, exc_info=e) break if len(data) == 0: logger.warning('Server returned an empty response') continue logger.info('Initializing Workbook...') progress['task'] = 'Task 2 of 4' progress['limit'] = len(data) progress['index'] = 0 t2.clear() t2.start() workbook = openpyxl.Workbook() sheet = workbook.active class Record: def __init__(self, fields: dict[int, str]): self.headers = fields self.data = dict() def clear(self): self.data.clear() def __setitem__(self, key, value): if key not in self.headers: raise KeyError(key) self.data[key] = value def __getitem__(self, key): if key not in self.headers: raise KeyError(key) return self.data[key] # Required fields # See record = Record({ 1: '订单号', 2: '订单名称', 3: '订单日期', 4: '当前处理人', 5: '业绩归属部门', 6: '客户编号', 7: '币种', 8: '产品名称', 9: '产品编号', 10: '产品型号', 11: '原价', 12: '折扣率', 13: '单价', 14: '数量', 15: '产品描述', 16: 'AVOIR', }) sheet.append(record.headers) categories = dict() clients = dict() try: for i, item in enumerate(data, 1): flow.do(Wait) flow.react() number: str = item['number'] logger.info('[%d/%d] Preprocessing data for %s', i, len(data), number) progress['number'] = number progress['index'] = i-1 if (category := categories.get(o := item['category_id'])) is None: if 'error' in (category := fetch(f'{base}/categories/{o}.json?api_token={profile.token}')): error = category['error'] code = category['code'] logger.warning("Error while fetching 'category' (code: %s, message: %s); skipping", code, error) continue if (client := clients.get(o := item['client_id'])) is None: if 'error' in (client := fetch(f'{base}/clients/{o}.json?api_token={profile.token}')): error = client['error'] code = client['code'] logger.warning("Error while fetching 'client' (code: %s, message: %s); skipping", code, error) continue identity = client['external_id'] if client['company'] else profile.person date: str = item['issue_date'] kind: str = item['kind'] total = float(item['price_net']) positions: list = item['positions'] for position in positions: code: str = position['code'] product: str = position['name'] description: str = position['description'] price = float(position['price_net'] or '0') discount = float(position['discount_percent'] or '0') quantity = float(position['quantity'] or '0') record.clear() record[1] = profile.format(number) record[2] = number.replace('/', '-') record[3] = date record[4] = category['name'] record[5] = profile.name record[6] = identity record[7] = 'USD' match kind: case 'vat': record[8] = product record[9] = (p.search('产品型号', code) or p.search('产品名称', product)).get('产品编号') record[10] = code record[11] = '%.2f' % price record[12] = '%g%%' % discount record[13] = '%.2f' % (price * (1 - discount / 100)) record[14] = '%g' % quantity record[15] = description case 'correction': record[8] = p.search('产品编号', profile.remise).get('产品名称') record[9] = profile.remise record[13] = '0' record[14] = '0' record[16] = '%.2f' % total positions.clear() if record[9] is None: logger.warning("Could not identify product '%s'", product) sheet.append(record.data) except Skip: pass except Cancel: continue except Exception as e: logger.error('Error while processing data', exc_info=e) status = Status.STANDBY continue try: filename = f'Order-Import-{profile.name}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}.xlsx' file = Path(parameters['directory']).joinpath(filename) logger.info('Saving excel file to %s', str(file)) flow.do(Wait) flow.react() workbook.save(file) except Skip: pass except Cancel: continue except Exception as e: logger.error('Error while saving excel file', exc_info=e) status = Status.STANDBY continue try: logger.info('Uploading...') progress.clear() progress['task'] = 'Task 3 of 4' t2.clear() t2.start() driver.switch_to.new_window('tab') driver.get(WEBURL % 'order/importOrder') click(".product-import-img-box .import-img-radio:nth-child(2) .mm-radio-group > label:nth-child(2) .mm-radio-input", condition=None) click(".product-import-img-box .mm-selector-rendered") click(".mm-outside.mm-select-dropdown ul li:nth-child(%d) span" % (1 if options.get('draft') else 6)) flow.do(Wait) flow.react() locate(".big-file-upload input", wait=False).send_keys(str(file)) click(".product-import-img-footer button") click(".product-import-img-footer button.mm-button__primary") while True: try: innerText = driver.find_element(By.CSS_SELECTOR, ".img-box-result-title").get_attribute('innerText') if innerText == '导入完成': break except: flow.do(Sleep) flow.react() click(".mm-notification-container .mm-icon-close") click(".product-import-img-footer button.mm-button__primary") err = locate(".mm-tbody table tbody tr:nth-child(1) td:nth-child(5) .okki-space-item:nth-child(1) button") if err.get_attribute('disabled') is None: logger.warning('Incomplete import detected; downloaded 1 related document') err.click() flow.do(Sleep) flow.react() click(".mm-tbody table tbody tr:nth-child(1) td:nth-child(3) a") flow.do(Sleep) flow.react() logger.info('Done') except Skip: pass except Cancel: continue except Exception as e: logger.error('Error while uploading excel file', exc_info=e) status = Status.STANDBY continue class Parse: def __init__(self, url: str): from urllib.parse import urlsplit, parse_qs parts = list(urlsplit(url)) query = parse_qs(parts[3]) self.parts = parts self.query = json.loads(query['query'][0]) if 'query' in query else dict() def encode(self, extra=None): from urllib.parse import urlencode, urlunsplit query = { 'query': json.dumps(self.query, separators=(',', ':')) } if extra is not None: query.update(extra) self.parts[3] = urlencode(query) return urlunsplit(self.parts) def get(self, key: str): return self.query[key] def set(self, key: str, value): self.query[key] = value progress['task'] = 'Task 4 of 4' progress['limit'] = len(data) t2.clear() t2.start() index = 0 attempts = 0 while index < len(data): try: attempts += 1 driver.switch_to.window(driver.window_handles[2]) item = data[index] kind = item['kind'] title = re.search(r'O\d+', item['title']) number = profile.format(item['number']) positions = item['positions'] opportunity = None if kind != 'vat': logger.info('[%d/%d] Skipping %s', index+1, len(data), number) index += 1 attempts = 0 continue if attempts > parameters['attempts']: logger.warning('Exhausted all allowed attempts; skipping %s', number) index += 1 attempts = 0 continue progress['number'] = number progress['index'] = index flow.allow(Cancel) flow.allow(Skip) flow.do(Wait) flow.react() try: for page in count(1): url = Parse(driver.current_url) url.set('page_size', 1) url.set('page', page) url.set('query_filters', [{ 'field': 'order_no', 'name': '订单号', 'operator': 'match', 'value': number, 'object_name': 'objOrder', 'field_type': 35, 'unit': '', 'array_flag': 0 }]) driver.get(url.encode()) flow.do(Wait) flow.do(Sleep) flow.react() url = Parse(driver.current_url) if url.get('page') != page: raise Exception(number) flow.do(Sleep) flow.react() link = locate(".virtual-list-wrap .vue-recycle-scroller .vue-recycle-scroller__item-wrapper > div:nth-child(1) .cell[data-cci='1'] a", wait=False) if link.text != number: continue logger.info('[%d/%d] Processing %s...', index+1, len(data), number) link.click() driver.switch_to.window(driver.window_handles[3]) flow.do(Wait) flow.react() click(".sticky .okki-space-item:nth-child(1) button") break except Skip: index += 1 attempts = 0 continue except Cancel: break except NoSuchElementException: logger.warning("Could not find invoice '%s'; skipping", number) index += 1 attempts = 0 continue except Exception as e: logger.error("Error while looking up invoice '%s'", number, exc_info=e) status = Status.STANDBY continue if title is not None and (match := title[0]): try: driver.switch_to.new_window('tab') url = Parse(WEBURL % 'crm/business/list') driver.get(url.encode({ 'mode': 'list' })) flow.do(Wait) flow.do(Sleep) flow.react() try: click(".new-wrapper .paas-next-invoice-list-filter-line-wrapper .okki-btn-background-ghost", wait=False) except: pass for page in count(1): url = Parse(WEBURL % 'crm/business/list') url.set('keyword', match) url.set('search_field', 'serial_keyword') url.set('curPage', page) url.set('pageSize', 1) driver.get(url.encode({ 'keyword': match, 'search_field': 'serial_keyword' })) flow.do(Wait) flow.do(Sleep) flow.react() url = Parse(driver.current_url) if url.get('curPage') != page: raise Exception(match) flow.do(Sleep) flow.react() cell = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(3) .ow-serial-read-pretty_ellipsis", wait=False) if cell.text != match: continue link = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(6) a", wait=False) opportunity = unicodedata.normalize('NFKD', link.get_attribute('title')) break except Skip: index += 1 attempts = 0 continue except Cancel: break except NoSuchElementException: logger.warning("Could not find opportunity '%s'; skipping", match) except Exception as e: logger.error("Error while looking up opportunity '%s'", match, exc_info=e) status = Status.STANDBY continue finally: driver.close() driver.switch_to.window(driver.window_handles[3]) if opportunity is not None: try: flow.do(Wait) flow.react() dropdown = locate("#rc_select_1") dropdown.clear() dropdown.send_keys(opportunity) menu = locate(".okki-select-dropdown") menuitems = menu.find_elements(By.CSS_SELECTOR, ".rc-virtual-list-holder-inner > div") for menuitem in menuitems: if menuitem.get_attribute('label').strip().startswith(opportunity): click(menuitem) except Skip: index += 1 attempts = 0 continue except Cancel: break except Exception as e: logger.warning('Could not select opportunity; skipping', exc_info=e) try: pagination = 10 click(".okki-pagination-options-size-changer") click(".okki-select-dropdown .rc-virtual-list-holder-inner > div:nth-child(1)") except: logger.warning('Unable to setup pagination; this may cause issues') try: ids = list() wrapper = locate(".paas-order-product-list .row-items", condition=None) for page in count(1): hits = 0 iteration = 0 flow.do(Wait) flow.react() while hits < pagination and iteration < parameters['attempts']: iteration += 1 height = int(wrapper.get_attribute('clientHeight')) if iteration > 1 else 0 driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height) rows = wrapper.find_elements(By.CSS_SELECTOR, ".row-item") for row in reversed(rows) if iteration > 1 else rows: flow.do(Wait) flow.react() driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height) serial = row.text.split('\n', 1)[0].strip() if not serial or serial in ids: continue driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) value = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='4'] input").get_attribute('value') driver.execute_script("arguments[0].scroll(400, 0);", wrapper) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) flow.do(Sleep) flow.react() target = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='6'] input") if (target.get_attribute('value') == '0'): target.send_keys(Keys.BACKSPACE) target.send_keys(value) ids.append(serial) hits += 1 if len(ids) >= len(positions): break button = locate(".text-right li.okki-pagination-next button", condition=None) if button.get_attribute('disabled') is not None and len(ids) < len(positions): raise Exception('Product list imcomplete; expected %d, got %d' % (len(positions), len(ids))) flow.do(Wait) flow.react() click(button) except Skip: index += 1 attempts = 0 continue except Cancel: break except Exception as e: logger.error('Error while modifying invoice', exc_info=e) status = Status.STANDBY continue try: click(".ow-box button.okki-btn-round", wait=False) flow.do(Sleep) flow.react() except Exception as e: logger.warning('Unable to unset additional fees; skipping', exc_info=e) try: flow.do(Wait) flow.react() flow.allow(Cancel, False) flow.allow(Skip, False) click(".sticky.bottom-0 button.okki-btn-primary", condition=None) flow.do(Sleep) flow.react() except Skip: pass except Cancel: break except Exception as e: logger.error('Error while saving document', exc_info=e) status = Status.STANDBY continue finally: driver.close() index += 1 attempts = 0 except Exception as e: logger.error('Unexpected error', exc_info=e) status = Status.STANDBY if __name__ == '__main__': try: logging.basicConfig(level=logging.INFO, format="[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M") logger = logging.getLogger() level = logging.getLevelNamesMapping().get(args.log_level, 'INFO') logger.setLevel(level) opts = ChromeOptions() opts.enable_downloads = True opts.add_argument('--deny-permission-prompts') opts.add_experimental_option('prefs', { 'download.default_directory': args.directory }) with keep.presenting(): driver = Chrome(options=opts) status = main(driver) except KeyboardInterrupt: status = 0 except Exception as e: logger.critical('Fatal error', exc_info=e) status = 1 finally: driver.quit() exit(status)