diff --git a/main.py b/main.py index cd72731..d36a93b 100644 --- a/main.py +++ b/main.py @@ -39,14 +39,14 @@ APIURL = "https://%s.vosfactures.fr" def main(driver: WebDriver, logger = logging.getLogger('main')): parameters = vars(args) http = PoolManager() - sp = ServiceProvider.default() - + sp = ServiceProvider(ServiceProvider.Options()) + class Status(Enum): BUSY = 0 READY = 1 RUNNING = 2 STANDBY = 3 - + class Profile: def __init__(self, name, subdomain, token, remise, person=None, prefix=None, suffix=None): self.name = name @@ -56,16 +56,16 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): self.person = person self.prefix = prefix self.suffix = suffix - + def format(self, number: str): result = ''.join(filter(bool, [self.prefix, number, self.suffix])) return result - + t1 = Timer() t2 = Timer() options = dict() status = Status.BUSY - + def begin(opts: dict, args: dict): nonlocal status options.update(opts) @@ -73,24 +73,24 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): parameters.update(args) t1.clear() t1.start() - + def pause(): nonlocal status status = Status.STANDBY t1.pause() t2.pause() - + def resume(): nonlocal status status = Status.RUNNING driver.switch_to.window(driver.current_window_handle) t1.start() t2.start() - + sp.add(begin, pause, resume) sp.set('status', lambda: status.name) sp.set('uptime', lambda: [t1.delta()]) - + try: profiles = [ Profile(**{ k.lower().strip(): v.strip() for k, v in map(lambda o: str.split(o, '=', 2), p) }) @@ -99,14 +99,14 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except Exception as e: logger.critical('Unable to load profiles', exc_info=e) return 2 - + try: sp.set('context', lambda: { 'profiles': list(map(vars, profiles)), 'parameters': parameters }) driver.get(sp.run()) except Exception as e: logger.critical('Unable to load starup page', exc_info=e) return 3 - + try: driver.switch_to.new_window('tab') driver.set_page_load_timeout(parameters['timeout']) @@ -114,11 +114,11 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except TimeoutException: logger.warning('Timeout') driver.execute_script("window.stop();") - + setup(driver, parameters) until(lambda x: 'loginProgress' in x.find_element(By.TAG_NAME, "body").get_attribute('class'), watch=False) logger.info('Waiting for authentication...') - + if (account := parameters['account']) and (password := parameters['password']): try: logger.info('Logging in as %s (%s)', str.split(account, '@', 1).pop(0).capitalize(), account) @@ -129,7 +129,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except Exception as e: logger.critical('Unable to login to %s', account, exc_info=e) return 3 - + while True: try: locate("#container", wait=False) @@ -137,30 +137,30 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): break except: sleep(parameters['interval']) - + class ProductInfo: def __init__(self, file): self.wb = openpyxl.load_workbook(file, read_only=True) self.indices = dict() self.headers = dict() - + def index(self, *keys: str): self.indices.clear() self.headers.clear() - + for col in range(1, self.wb.active.max_column + 1): value = str(self.wb.active.cell(1, col).value) self.headers[value] = col - + for row in range(1, self.wb.active.max_row + 1): for key, col in map(lambda k: (k, self.headers[k]), keys): value = str(self.wb.active.cell(row, col).value) self.indices.setdefault(key, dict())[value] = row - + def at(self, axis: str, index) -> dict: if (a := self.indices.get(axis)) is None or (b := dict.get(a, index)) is None: return dict() return { k: self.wb.active.cell(b, c).value for k, c in self.headers.items() } - + try: logger.info('Downloading product list...') click("header .okki-space .okki-space-item:nth-child(1) button") @@ -169,7 +169,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): click(".okki-modal.product-export-wrap .mm-selector-rendered") click(".mm-outside.ui-field-selector-popper .selector-area:nth-child(1) button") click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary") - + while True: try: click(".okki-modal.product-export-wrap .okki-modal-footer button.okki-btn-primary") @@ -179,7 +179,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): break except: sleep(parameters['interval']) - + file = Path(parameters['directory']).joinpath(filename) until(lambda _: file.exists(), watch=False) p = ProductInfo(file) @@ -192,7 +192,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except Exception as e: logger.critical('Unable to load products', exc_info=e) return 4 - + def fetch(url: str, method = 'GET', retry = parameters['attempts']): for attempt in range(1, retry + 1): try: @@ -202,61 +202,61 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except Exception as e: logger.warning('Error while fetching data from %s, retrying... (%d)', url, attempt, exc_info=e) assert attempt < retry, "Exceeded maximum retry attempts" - + class Wait(Action): @classmethod def prepare(cls): return True - + @classmethod def perform(cls): if status == Status.RUNNING: return False sleep(0.2); return True - + class Sleep(Action): @classmethod def prepare(cls): return True - + @classmethod def perform(cls): sleep(parameters['interval']) return False - + class Cancel(Action): @classmethod def prepare(cls): nonlocal status status = Status.RUNNING return True - + @classmethod def perform(cls): nonlocal status status = Status.READY driver.switch_to.window(driver.window_handles[0]) raise cls - + class Skip(Action): @classmethod def prepare(cls): return True - + @classmethod def perform(cls): driver.switch_to.window(driver.current_window_handle) raise cls - + flow = ActionFlow() flow.stage(Wait, Sleep, Cancel, Skip) profile = None progress = { 'task': '' } selection = 0 - + sp.add(pairs=[ (k.lower(), v) for k, v in flow ]) sp.set('actions', lambda: flow.capabilities()) sp.set('progress', lambda: progress) - + while not flow.react(Wait): try: for i in range(len(driver.window_handles), 1, -1): @@ -281,33 +281,33 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error('Unexpected error', exc_info=e) status = Status.STANDBY continue - + progress.clear() progress['task'] = 'Task 1 of 4' t2.clear() t2.start() sp.pop('uptime') sp.set('uptime', lambda: [t1.delta(), t2.delta()]) - + base = APIURL % profile.subdomain data = list() df = options.get('datefrom') dt = options.get('dateto') types = ['vat'] if options.get('avoir'): types.append('correction') - + logger.info('Profile selected: %s', profile.name) logger.info('Date from %s to %s', df, dt) flow.allow(Cancel) flow.deter(Skip) - + for page in count(1): try: result = fetch(f'{base}/invoices.json?{'&'.join([f'kinds%5B%5D={k}' for k in types])}&api_token={profile.token}&include_positions=true&per_page=25&page={page}&period=more&date_from={df}&date_to={dt}') if 'message' in result: raise Exception(result['message']) if not isinstance(result, list): raise TypeError() if len(result) == 0: break - + logger.info('Downloading invoices (%d)', page) data.extend(result) flow.react(Wait, Sleep) @@ -318,11 +318,11 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except Exception as e: logger.error('Error while fetching data from %s', base, exc_info=e) break - + if len(data) == 0: logger.warning('Server returned an empty response') continue - + logger.info('Initializing Workbook...') progress['task'] = 'Task 2 of 4' progress['limit'] = len(data) @@ -331,23 +331,23 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): t2.start() workbook = openpyxl.Workbook() sheet = workbook.active - + class Record: def __init__(self, fields: dict[int, str]): self.headers = fields self.data = dict() - + def clear(self): self.data.clear() - + def __setitem__(self, key, value): if key not in self.headers: raise KeyError(key) self.data[key] = value - + def __getitem__(self, key): if key not in self.headers: raise KeyError(key) return self.data[key] - + # Required fields # See record = Record({ @@ -371,7 +371,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): sheet.append(record.headers) categories = dict() clients = dict() - + try: for i, item in enumerate(data, 1): flow.react(Wait) @@ -379,27 +379,27 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.info('[%d/%d] Preprocessing data for %s', i, len(data), number) progress['number'] = number progress['index'] = i-1 - + if (category := categories.get(o := item['category_id'])) is None: if 'error' in (category := fetch(f'{base}/categories/{o}.json?api_token={profile.token}')): error = dict.get(category, 'error') response = dict.get(category, 'status') logger.warning("Error while fetching field 'category' (status: %s, message: %s); skipping", response, error) continue - + if (client := clients.get(o := item['client_id'])) is None: if 'error' in (client := fetch(f'{base}/clients/{o}.json?api_token={profile.token}')): error = dict.get(category, 'error') response = dict.get(category, 'status') logger.warning("Error while fetching field 'client' (status: %s, message: %s); skipping", response, error) continue - + identity = client['shortcut'] if client['company'] else profile.person date: str = item['issue_date'] kind: str = item['kind'] total = float(item['price_net']) positions: list = item['positions'] - + for position in positions: code: str = position['code'] product: str = position['name'] @@ -407,7 +407,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): price = float(position['price_net'] or '0') discount = float(position['discount_percent'] or '0') quantity = float(position['quantity'] or '0') - + record.clear() record[1] = profile.format(number) record[2] = number.replace('/', '-') @@ -416,7 +416,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): record[5] = profile.name record[6] = identity record[7] = 'USD' - + match kind: case 'vat': record[8] = product @@ -434,7 +434,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): record[14] = '0' record[16] = '%.2f' % total positions.clear() - + if record[9] is None: logger.warning("Could not identify product '%s'", product) sheet.append(record.data) except Skip: @@ -445,11 +445,11 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error('Error while processing data', exc_info=e) status = Status.STANDBY continue - + if sheet.max_row < 2: logger.warning("No data; skipping") continue - + try: filename = f'Order-Import-{profile.name}-{datetime.now().strftime('%Y%m%d-%H%M%S-%f')}.xlsx' file = Path(parameters['directory']).joinpath(filename) @@ -464,7 +464,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error('Error while saving document', exc_info=e) status = Status.STANDBY continue - + try: logger.info('Uploading data...') progress.clear() @@ -473,7 +473,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): t2.start() driver.switch_to.new_window('tab') driver.get(WEBURL % 'order/importOrder') - + click(".product-import-img-box .import-img-radio:nth-child(2) .mm-radio-group > label:nth-child(2) .mm-radio-input", condition=None) click(".product-import-img-box .mm-selector-rendered") click(".mm-outside.mm-select-dropdown ul li:nth-child(%d) span" % (1 if options.get('draft') else 6)) @@ -488,7 +488,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error('Error while uploading document', exc_info=e) status = Status.STANDBY continue - + while not flow.react(Wait, Sleep): try: err = locate(".mm-tbody table tbody tr:nth-child(1) td:nth-child(5) .okki-space-item:nth-child(1) button", wait=False) @@ -498,16 +498,16 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): except: pass try: click(".list-header-wrap button", wait=False) except: pass - + if err.get_attribute('disabled') is None: click(err, condition=None) logger.warning('Incomplete import detected; downloading 1 related document') flow.react(Sleep) - + try: click(".mm-tbody table tbody tr:nth-child(1) td:nth-child(3) a") except: continue flow.react(Sleep) - + class Parse: def __init__(self, url: str): from urllib.parse import urlsplit, parse_qs @@ -515,27 +515,27 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): query = parse_qs(parts[3]) self.parts = parts self.query = json.loads(query['query'][0]) if 'query' in query else dict() - + def encode(self, extra=None): from urllib.parse import urlencode, urlunsplit query = { 'query': json.dumps(self.query, separators=(',', ':')) } if extra is not None: query.update(extra) self.parts[3] = urlencode(query) return urlunsplit(self.parts) - + def get(self, key: str): return self.query[key] - + def set(self, key: str, value): self.query[key] = value - + progress['task'] = 'Task 4 of 4' progress['limit'] = len(data) t2.clear() t2.start() index = 0 attempts = 0 - + while index < len(data): try: attempts += 1 @@ -547,20 +547,20 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): number = profile.format(item['number']) positions = item['positions'] opportunity = None - + if kind != 'vat': logger.info('[%d/%d] Undesired invoice type; skipping %s', index+1, len(data), number) raise Skip() - + if attempts > parameters['attempts']: logger.warning('Exhausted all allowed attempts; skipping %s', number) raise Skip() - + progress['number'] = number progress['index'] = index flow.allow(Skip, Cancel) flow.react(Wait) - + try: for page in count(1): url = Parse(driver.current_url) @@ -580,7 +580,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): flow.react(Wait, Sleep) url = Parse(driver.current_url) if url.get('page') != page: raise Exception(number) - + flow.react(Sleep) link = locate(".virtual-list-wrap .vue-recycle-scroller .vue-recycle-scroller__item-wrapper > div:nth-child(1) .cell[data-cci='1'] a", wait=False) if link.text != number: continue @@ -599,7 +599,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error("Error while looking up invoice '%s'", number, exc_info=e) status = Status.STANDBY continue - + if title is not None and (match := title[0]): try: base = WEBURL % 'crm/business/list' @@ -607,10 +607,10 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): url = Parse(base) driver.get(url.encode({ 'mode': 'list' })) flow.react(Wait, Sleep) - + try: click(".new-wrapper .paas-next-invoice-list-filter-line-wrapper .okki-btn-background-ghost", wait=False) except: pass - + for page in count(1): url = Parse(base) url.set('keyword', match) @@ -621,7 +621,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): flow.react(Wait, Sleep) url = Parse(driver.current_url) if url.get('curPage') != page: raise Exception(match) - + flow.react(Sleep) cell = locate(".virtual-list-wrap .vue-recycle-scroller .row-item > .cell:nth-child(3) .ow-serial-read-pretty_ellipsis", wait=False) if cell.text != match: continue @@ -639,20 +639,20 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): finally: driver.close() driver.switch_to.window(driver.window_handles[3]) - + if opportunity is not None: try: flow.react(Wait) label = locate("label.paas-form-item-label") dropdown = locate("label.paas-form-item-label[title='商机'] + div input") - + for iteration in count(1): dropdown.click() dropdown.clear() dropdown.send_keys(opportunity) menu = locate(".okki-select-dropdown") menuitems = menu.find_elements(By.CSS_SELECTOR, ".rc-virtual-list-holder-inner > div") - + try: for menuitem in menuitems: if menuitem.get_attribute('label') == opportunity: @@ -666,37 +666,37 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): raise e except Exception as e: logger.warning("Could not select opportunity '%s'", opportunity, exc_info=e) - + try: pagination = 10 click(".okki-pagination-options-size-changer") click(".okki-select-dropdown .rc-virtual-list-holder-inner > div:nth-child(1)") except: logger.warning('Unable to setup pagination; this may cause issues') - + try: ids = list() wrapper = locate(".paas-order-product-list .row-items", condition=None) - + for page in count(1): hits = 0 iteration = 0 flow.react(Wait) - + while hits < pagination and iteration < parameters['attempts']: iteration += 1 height = int(wrapper.get_attribute('clientHeight')) if iteration > 1 else 0 driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height) rows = wrapper.find_elements(By.CSS_SELECTOR, ".row-item") - + for row in reversed(rows) if iteration > 1 else rows: flow.react(Wait) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) driver.execute_script("arguments[0].scrollTo(0, arguments[1]);", wrapper, height) serial = row.text.split('\n', 1)[0].strip() if not serial or serial in ids: continue - + driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", row) driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) value = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='4'] input").get_attribute('value') @@ -705,14 +705,14 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): driver.execute_script("arguments[0].scrollIntoView({ block: 'center' });", wrapper) flow.react(Sleep) target = row.find_element(By.CSS_SELECTOR, ".cell[data-cci='6'] input") - + if (target.get_attribute('value') == '0'): target.send_keys(Keys.BACKSPACE) target.send_keys(value) - + ids.append(serial) hits += 1 - + if len(ids) >= len(positions): break button = locate(".text-right li.okki-pagination-next button", condition=None) if button.get_attribute('disabled') is not None and len(ids) < len(positions): @@ -725,13 +725,13 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error('Error while modifying invoice', exc_info=e) status = Status.STANDBY continue - + try: click(".ow-box button.okki-btn-round", wait=False) flow.react(Sleep) except Exception as e: logger.warning('Unable to unset additional fees', exc_info=e) - + flow.react(Wait) flow.deter(Skip, Cancel) click(".sticky.bottom-0 button.okki-btn-primary", condition=None) @@ -745,7 +745,7 @@ def main(driver: WebDriver, logger = logging.getLogger('main')): logger.error('Unexpected error', exc_info=e) status = Status.STANDBY continue - + index += 1 attempts = 0 @@ -755,12 +755,12 @@ if __name__ == '__main__': logger = logging.getLogger() level = logging.getLevelNamesMapping().get(args.log_level, 'INFO') logger.setLevel(level) - + opts = ChromeOptions() opts.enable_downloads = True opts.add_argument('--deny-permission-prompts') opts.add_experimental_option('prefs', { 'download.default_directory': args.directory }) - + with keep.presenting(): driver = Chrome(options=opts) status = main(driver) diff --git a/requirements.txt b/requirements.txt index f86882b..a33245d 100644 Binary files a/requirements.txt and b/requirements.txt differ