refactor: improve deep methods for better performance

dev_tools/button_extract.py

@@ -11,8 +11,9 @@ from module.base.code_generator import CodeGenerator
 from module.base.utils import SelectedGrids, area_center, area_limit, area_pad, corner2area, get_bbox, get_color, \
     image_size, load_image
 from module.config.config_manual import ManualConfig as AzurLaneConfig
+from module.config.deep import deep_get, deep_iter, deep_set
 from module.config.server import VALID_LANG
-from module.config.utils import deep_get, deep_iter, deep_set, iter_folder
+from module.config.utils import iter_folder
 from module.logger import logger
 
 SHARE_SERVER = 'share'

module/alas.py

@@ -10,7 +10,7 @@ from cached_property import cached_property
 from MCE.custom_widgets.ctkmessagebox import CTkMessagebox
 from module.base.decorator import del_cached_property
 from module.config.config import AzurLaneConfig, TaskEnd
-from module.config.utils import deep_get, deep_set
+from module.config.deep import deep_get, deep_set
 from module.exception import *
 from module.logger import logger, save_error_log
 from module.notify import handle_notify

module/config/config.py

@@ -1,16 +1,17 @@
 import copy
-import datetime
 import operator
 import threading
+from datetime import datetime, timedelta
 
 from module.base.decorator import cached_property, del_cached_property
 from module.base.filter import Filter
 from module.config.config_generated import GeneratedConfig
 from module.config.config_manual import ManualConfig
-from module.config.config_updater import ConfigUpdater
+from module.config.config_updater import ConfigUpdater, ensure_time, get_server_next_update, nearest_future
+from module.config.deep import deep_get, deep_set
 from module.config.stored.classes import iter_attribute
 from module.config.stored.stored_generated import StoredGenerated
-from module.config.utils import *
+from module.config.utils import DEFAULT_TIME, dict_to_kv, filepath_config, path_to_arg
 from module.config.watcher import ConfigWatcher
 from module.exception import RequestHumanTakeover, ScriptError
 from module.logger import logger

module/config/config_updater.py

@@ -5,6 +5,7 @@ from cached_property import cached_property
 
 from deploy.Windows.utils import DEPLOY_TEMPLATE, poor_yaml_read, poor_yaml_write
 from module.base.timer import timer
+from module.config.deep import deep_default, deep_get, deep_iter, deep_set
 from module.config.server import VALID_SERVER
 from module.config.utils import *
 

module/config/deep.py

@@ -0,0 +1,533 @@
+from collections import deque
+
+# deep_* functions are used for access nested dictionary.
+# They target for high performance so code are complicated to read
+# In general performance practise, time costs are as below:
+# - When key exists
+#   try: dict[key] except KeyError << dict.get(key) < if key in dict: dict[key]
+# - When not key exists
+#   if key in dict: dict[key] < dict.get(key) <<< try: dict[key] except KeyError
+
+OP_ADD = 'add'
+OP_SET = 'set'
+OP_DEL = 'del'
+
+
+def deep_get(d, keys, default=None):
+    """
+    Get value from nested dict and list
+    https://stackoverflow.com/questions/25833613/safe-method-to-get-value-of-nested-dictionary
+
+    Args:
+        d (dict):
+        keys (list[str], str): Such as ['Scheduler', 'NextRun', 'value']
+        default: Default return if key not found.
+
+    Returns:
+        Value on given keys
+    """
+    # 240 + 30 * depth (ns)
+    if type(keys) is str:
+        keys = keys.split('.')
+
+    try:
+        for k in keys:
+            d = d[k]
+        return d
+    # No such key
+    except KeyError:
+        return default
+    # No such key
+    except IndexError:
+        return default
+    # Input `keys` is not iterable or input `d` is not dict
+    # list indices must be integers or slices, not str
+    except TypeError:
+        return default
+
+
+def deep_get_with_error(d, keys):
+    """
+    Get value from nested dict and list, raise KeyError if key not exists
+
+    Args:
+        d (dict):
+        keys (list[str], str): Such as ['Scheduler', 'NextRun', 'value']
+
+    Returns:
+        Value on given keys
+
+    Raises:
+        KeyError: If key not exists
+    """
+    # 240 + 30 * depth (ns)
+    if type(keys) is str:
+        keys = keys.split('.')
+
+    try:
+        for k in keys:
+            d = d[k]
+        return d
+    # No such key
+    # except KeyError:
+    #     raise
+    # No such key
+    except IndexError:
+        raise KeyError
+    # Input `keys` is not iterable or input `d` is not dict
+    # list indices must be integers or slices, not str
+    except TypeError:
+        raise KeyError
+
+
+def deep_exist(d, keys):
+    """
+    Check if keys exists in nested dict or list
+
+    Args:
+        d (dict):
+        keys (str, list): Such as `Scheduler.NextRun.value`
+
+    Returns:
+        bool: If key exists
+    """
+    # 240 + 30 * depth (ns)
+    if type(keys) is str:
+        keys = keys.split('.')
+
+    try:
+        for k in keys:
+            d = d[k]
+        return True
+    # No such key
+    except KeyError:
+        return False
+    # No such key
+    except IndexError:
+        return False
+    # Input `keys` is not iterable or input `d` is not dict
+    # list indices must be integers or slices, not str
+    except TypeError:
+        return False
+
+
+def deep_set(d, keys, value):
+    """
+    Set value into nested dict safely, imitating deep_get().
+    Can only set dict
+    """
+    # 150 * depth (ns)
+    if type(keys) is str:
+        keys = keys.split('.')
+
+    first = True
+    exist = True
+    prev_d = None
+    prev_k = None
+    prev_k2 = None
+    try:
+        for k in keys:
+            if first:
+                prev_d = d
+                prev_k = k
+                first = False
+                continue
+            try:
+                # if key in dict: dict[key] > dict.get > dict.setdefault > try dict[key] except
+                if exist and prev_k in d:
+                    prev_d = d
+                    d = d[prev_k]
+                else:
+                    exist = False
+                    new = {}
+                    d[prev_k] = new
+                    d = new
+            except TypeError:
+                # `d` is not dict
+                exist = False
+                d = {}
+                prev_d[prev_k2] = {prev_k: d}
+
+            prev_k2 = prev_k
+            prev_k = k
+            # prev_k2, prev_k = prev_k, k
+    # Input `keys` is not iterable
+    except TypeError:
+        return
+
+    # Last key, set value
+    try:
+        d[prev_k] = value
+        return
+    # Last value `d` is not dict
+    except TypeError:
+        prev_d[prev_k2] = {prev_k: value}
+        return
+
+
+def deep_default(d, keys, value):
+    """
+    Set value into nested dict safely, imitating deep_get().
+    Can only set dict
+    """
+    # 150 * depth (ns)
+    if type(keys) is str:
+        keys = keys.split('.')
+
+    first = True
+    exist = True
+    prev_d = None
+    prev_k = None
+    prev_k2 = None
+    try:
+        for k in keys:
+            if first:
+                prev_d = d
+                prev_k = k
+                first = False
+                continue
+            try:
+                # if key in dict: dict[key] > dict.get > dict.setdefault > try dict[key] except
+                if exist and prev_k in d:
+                    prev_d = d
+                    d = d[prev_k]
+                else:
+                    exist = False
+                    new = {}
+                    d[prev_k] = new
+                    d = new
+            except TypeError:
+                # `d` is not dict
+                exist = False
+                d = {}
+                prev_d[prev_k2] = {prev_k: d}
+
+            prev_k2 = prev_k
+            prev_k = k
+            # prev_k2, prev_k = prev_k, k
+    # Input `keys` is not iterable
+    except TypeError:
+        return
+
+    # Last key, set value
+    try:
+        d.setdefault(prev_k, value)
+        return
+    # Last value `d` is not dict
+    except AttributeError:
+        prev_d[prev_k2] = {prev_k: value}
+        return
+
+
+def deep_pop(d, keys, default=None):
+    """
+    Pop value from nested dict and list
+    """
+    if type(keys) is str:
+        keys = keys.split('.')
+
+    try:
+        for k in keys[:-1]:
+            d = d[k]
+        # No `pop(k, default)` so it can pop list
+        return d.pop(keys[-1])
+    # No such key
+    except KeyError:
+        return default
+    # Input `keys` is not iterable or input `d` is not dict
+    # list indices must be integers or slices, not str
+    except TypeError:
+        return default
+    # Input `keys` out of index
+    except IndexError:
+        return default
+    # Last `d` is not dict
+    except AttributeError:
+        return default
+
+
+def deep_iter_depth1(data):
+    """
+    Equivalent to data.items() but suppress error if data is not a dict
+
+    Args:
+        data:
+
+    Yields:
+        Any: Key
+        Any: Value
+    """
+    try:
+        for k, v in data.items():
+            yield k, v
+        return
+    except AttributeError:
+        # `data` is not dict
+        return
+
+
+def deep_iter_depth2(data):
+    """
+    Iter key and value in nested dict of depth 2
+    A simplified deep_iter
+
+    Args:
+        data:
+
+    Yields:
+        Any: Key1
+        Any: Key2
+        Any: Value
+    """
+    try:
+        for k1, v1 in data.items():
+            if type(v1) is dict:
+                for k2, v2 in v1.items():
+                    yield k1, k2, v2
+    except AttributeError:
+        # `data` is not dict
+        return
+
+
+def deep_iter(data, min_depth=None, depth=3):
+    """
+    Iter key and value in nested dict
+    300us on alas.json depth=3 (530+ rows)
+    Can only iter dict
+
+    Args:
+        data:
+        min_depth:
+        depth:
+
+    Yields:
+        list[str]: Key path
+        Any: Value
+    """
+    if min_depth is None:
+        min_depth = depth
+    assert 1 <= min_depth <= depth
+
+    # Equivalent to dict.items()
+    try:
+        if depth == 1:
+            for k, v in data.items():
+                yield [k], v
+            return
+        # Iter first depth
+        elif min_depth == 1:
+            q = deque()
+            for k, v in data.items():
+                key = [k]
+                if type(v) is dict:
+                    q.append((key, v))
+                else:
+                    yield key, v
+        # Iter target depth only
+        else:
+            q = deque()
+            for k, v in data.items():
+                key = [k]
+                if type(v) is dict:
+                    q.append((key, v))
+    except AttributeError:
+        # `data` is not dict
+        return
+
+    # Iter depths
+    current = 2
+    while current <= depth:
+        new_q = deque()
+        # max depth
+        if current == depth:
+            for key, data in q:
+                for k, v in data.items():
+                    yield key + [k], v
+        # in target depth
+        elif min_depth <= current < depth:
+            for key, data in q:
+                for k, v in data.items():
+                    subkey = key + [k]
+                    if type(v) is dict:
+                        new_q.append((subkey, v))
+                    else:
+                        yield subkey, v
+        # Haven't reached min depth
+        else:
+            for key, data in q:
+                for k, v in data.items():
+                    subkey = key + [k]
+                    if type(v) is dict:
+                        new_q.append((subkey, v))
+        q = new_q
+        current += 1
+
+
+def deep_values(data, min_depth=None, depth=3):
+    """
+    Iter value in nested dict
+    300us on alas.json depth=3 (530+ rows)
+    Can only iter dict
+
+    Args:
+        data:
+        min_depth:
+        depth:
+
+    Yields:
+        Any: Value
+    """
+    if min_depth is None:
+        min_depth = depth
+    assert 1 <= min_depth <= depth
+
+    # Equivalent to dict.items()
+    try:
+        if depth == 1:
+            for v in data.values():
+                yield v
+            return
+        # Iter first depth
+        elif min_depth == 1:
+            q = deque()
+            for v in data.values():
+                if type(v) is dict:
+                    q.append(v)
+                else:
+                    yield v
+        # Iter target depth only
+        else:
+            q = deque()
+            for v in data.values():
+                if type(v) is dict:
+                    q.append(v)
+    except AttributeError:
+        # `data` is not dict
+        return
+
+    # Iter depths
+    current = 2
+    while current <= depth:
+        new_q = deque()
+        # max depth
+        if current == depth:
+            for data in q:
+                for v in data.values():
+                    yield v
+        # in target depth
+        elif min_depth <= current < depth:
+            for data in q:
+                for v in data.values():
+                    if type(v) is dict:
+                        new_q.append(v)
+                    else:
+                        yield v
+        # Haven't reached min depth
+        else:
+            for data in q:
+                for v in data.values():
+                    if type(v) is dict:
+                        new_q.append(v)
+        q = new_q
+        current += 1
+
+
+def deep_iter_diff(before, after):
+    """
+    Iter diff between 2 dict.
+    Pretty fast to compare 2 deeply nested dict,
+    time cost increases with the number of differences.
+
+    Args:
+        before:
+        after:
+
+    Yields:
+        list[str]: Key path
+        Any: Value in before, or None if not exists
+        Any: Value in after, or None if not exists
+    """
+    if before == after:
+        return
+    if type(before) is not dict or type(after) is not dict:
+        yield [], before, after
+        return
+
+    queue = deque([([], before, after)])
+    while True:
+        new_queue = deque()
+        for path, d1, d2 in queue:
+            keys1 = set(d1.keys())
+            keys2 = set(d2.keys())
+            for key in keys1.union(keys2):
+                try:
+                    val2 = d2[key]
+                except KeyError:
+                    # Safe to access d1[key], because key came from the union of both
+                    # If it's not in d2 then it's in d1
+                    yield path + [key], d1[key], None
+                    continue
+                try:
+                    val1 = d1[key]
+                except KeyError:
+                    yield path + [key], None, val2
+                    continue
+                # Compare dict first, which is pretty fast
+                if val1 != val2:
+                    if type(val1) is dict and type(val2) is dict:
+                        new_queue.append((path + [key], val1, val2))
+                    else:
+                        yield path + [key], val1, val2
+        queue = new_queue
+        if not queue:
+            break
+
+
+def deep_iter_patch(before, after):
+    """
+    Iter patch event from before to after, like creating a json-patch
+    Pretty fast to compare 2 deeply nested dict,
+    time cost increases with the number of differences.
+
+    Args:
+        before:
+        after:
+
+    Yields:
+        str: OP_ADD, OP_SET, OP_DEL
+        list[str]: Key path
+        Any: Value in after,
+            or None of event is OP_DEL
+    """
+    if before == after:
+        return
+    if type(before) is not dict or type(after) is not dict:
+        yield OP_SET, [], after
+        return
+
+    queue = deque([([], before, after)])
+    while True:
+        new_queue = deque()
+        for path, d1, d2 in queue:
+            keys1 = set(d1.keys())
+            keys2 = set(d2.keys())
+            for key in keys1.union(keys2):
+                try:
+                    val2 = d2[key]
+                except KeyError:
+                    yield OP_DEL, path + [key], None
+                    continue
+                try:
+                    val1 = d1[key]
+                except KeyError:
+                    yield OP_ADD, path + [key], val2
+                    continue
+                # Compare dict first, which is pretty fast
+                if val1 != val2:
+                    if type(val1) is dict and type(val2) is dict:
+                        new_queue.append((path + [key], val1, val2))
+                    else:
+                        yield OP_SET, path + [key], val2
+        queue = new_queue
+        if not queue:
+            break

module/config/stored/classes.py

@@ -2,7 +2,8 @@ from datetime import datetime
 from functools import cached_property as functools_cached_property
 
 from module.base.decorator import cached_property
-from module.config.utils import DEFAULT_TIME, deep_get, get_server_last_monday_update, get_server_last_update
+from module.config.deep import deep_get
+from module.config.utils import DEFAULT_TIME, get_server_last_monday_update, get_server_last_update
 
 
 # from module.exception import ScriptError

module/config/utils.py

@@ -2,7 +2,6 @@ import json
 import os
 import random
 import string
-from collections import deque
 from datetime import datetime, timedelta, timezone
 
 import yaml
@@ -171,188 +170,6 @@ def alas_instance():
     return out
 
 
-def deep_get(d, keys, default=None):
-    # 240 + 30 * depth (ns)
-    if type(keys) is str:
-        keys = keys.split('.')
-
-    try:
-        for k in keys:
-            d = d[k]
-        return d
-    # No such key
-    except KeyError:
-        return default
-    # Input `keys` is not iterable or input `d` is not dict
-    except TypeError:
-        return default
-
-
-def deep_set(d, keys, value):
-    """
-    Set value into dictionary safely, imitating deep_get().
-    """
-    # 150 * depth (ns)
-    if type(keys) is str:
-        keys = keys.split('.')
-
-    first = True
-    exist = True
-    prev_d = None
-    prev_k = None
-    prev_k2 = None
-    try:
-        for k in keys:
-            if first:
-                prev_d = d
-                prev_k = k
-                first = False
-                continue
-            try:
-                # if key in dict: dict[key] > dict.get > dict.setdefault > try dict[key] except
-                if exist and prev_k in d:
-                    prev_d = d
-                    d = d[prev_k]
-                else:
-                    exist = False
-                    new = {}
-                    d[prev_k] = new
-                    d = new
-            except TypeError:
-                # `d` is not dict
-                exist = False
-                d = {}
-                prev_d[prev_k2] = {prev_k: d}
-
-            prev_k2 = prev_k
-            prev_k = k
-            # prev_k2, prev_k = prev_k, k
-    # Input `keys` is not iterable
-    except TypeError:
-        return
-
-    # Last key, set value
-    try:
-        d[prev_k] = value
-        return
-    # Last value `d` is not dict
-    except TypeError:
-        prev_d[prev_k2] = {prev_k: value}
-        return
-
-
-def deep_pop(d, keys, default=None):
-    if type(keys) is str:
-        keys = keys.split('.')
-
-    try:
-        for k in keys[:-1]:
-            d = d[k]
-        return d.pop(keys[-1], default)
-    # No such key
-    except KeyError:
-        return default
-    # Input `keys` is not iterable or input `d` is not dict
-    except TypeError:
-        return default
-    # Input `keys` out of index
-    except IndexError:
-        return default
-    # Last `d` is not dict
-    except AttributeError:
-        return default
-
-
-def deep_default(d, keys, value):
-    """
-    Set default value into dictionary safely, imitating deep_get().
-    Value is set only when the dict doesn't contain such keys.
-    """
-    if isinstance(keys, str):
-        keys = keys.split('.')
-    assert type(keys) is list
-    if not keys:
-        if d:
-            return d
-        else:
-            return value
-    if not isinstance(d, dict):
-        d = {}
-    d[keys[0]] = deep_default(d.get(keys[0], {}), keys[1:], value)
-    return d
-
-
-def deep_iter(data, min_depth=None, depth=3):
-    """
-    300us on alas.json depth=3 (530+ rows)
-
-    Args:
-        data:
-        min_depth:
-        depth:
-
-    Returns:
-
-    """
-    if min_depth is None:
-        min_depth = depth
-    assert 1 <= min_depth <= depth
-
-    # Equivalent to dict.items()
-    try:
-        if depth == 1:
-            for k, v in data.items():
-                yield [k], v
-            return
-        # Iter first depth
-        elif min_depth == 1:
-            q = deque()
-            for k, v in data.items():
-                key = [k]
-                if type(v) is dict:
-                    q.append((key, v))
-                else:
-                    yield key, v
-        # Iter target depth only
-        else:
-            q = deque()
-            for k, v in data.items():
-                key = [k]
-                if type(v) is dict:
-                    q.append((key, v))
-    except AttributeError:
-        # `data` is not dict
-        return
-
-    # Iter depths
-    current = 2
-    while current <= depth:
-        new_q = deque()
-        # max depth
-        if current == depth:
-            for key, data in q:
-                for k, v in data.items():
-                    yield key + [k], v
-        # in target depth
-        elif min_depth <= current < depth:
-            for key, data in q:
-                for k, v in data.items():
-                    subkey = key + [k]
-                    if type(v) is dict:
-                        new_q.append((subkey, v))
-                    else:
-                        yield subkey, v
-        # Haven't reached min depth
-        else:
-            for key, data in q:
-                for k, v in data.items():
-                    subkey = key + [k]
-                    if type(v) is dict:
-                        new_q.append((subkey, v))
-        q = new_q
-        current += 1
-
-
 def parse_value(value, data):
     """
     Convert a string to float, int, datetime, if possible.

module/device/method/nemu_ipc.py

@@ -11,7 +11,7 @@ import numpy as np
 from module.base.decorator import cached_property, del_cached_property, has_cached_property
 from module.base.timer import Timer
 from module.base.utils import ensure_time
-from module.config.utils import deep_get
+from module.config.deep import deep_get
 from module.device.env import IS_WINDOWS
 from module.device.method.minitouch import insert_swipe, random_rectangle_point
 from module.device.method.pool import JobTimeout, WORKER_POOL

module/webui/app.py

@@ -34,12 +34,10 @@ from pywebio.session import go_app, info, local, register_thread, run_js, set_en
 
 import module.webui.lang as lang
 from module.config.config import AzurLaneConfig, Function
+from module.config.deep import deep_get, deep_iter, deep_set
 from module.config.utils import (
     alas_instance,
     alas_template,
-    deep_get,
-    deep_iter,
-    deep_set,
     dict_to_kv,
     filepath_args,
     filepath_config,

module/webui/lang.py

@@ -1,7 +1,9 @@
 import time
+from datetime import datetime
 from typing import Dict
 
-from module.config.utils import *
+from module.config.deep import deep_iter
+from module.config.utils import DEFAULT_TIME, LANGUAGES, filepath_i18n, read_file
 from module.webui.fake import list_mod
 from module.webui.setting import State
 

module/webui/translate.py

@@ -6,8 +6,8 @@ from pywebio.output import put_buttons, put_markdown
 from pywebio.session import defer_call, hold, run_js, set_env
 
 import module.webui.lang as lang
-from module.config.utils import (LANGUAGES, deep_get, deep_iter, deep_set,
+from module.config.deep import deep_get, deep_iter, deep_set
-                                 filepath_i18n, read_file, write_file)
+from module.config.utils import LANGUAGES, filepath_i18n, read_file, write_file
 
 
 def translate():

module/webui/utils.py

@@ -15,7 +15,7 @@ from pywebio.session import eval_js, info as session_info, register_thread, run_
 from rich.console import Console
 from rich.terminal_theme import TerminalTheme
 
-from module.config.utils import deep_iter
+from module.config.deep import deep_iter
 from module.logger import logger
 from module.webui.setting import State