Airtest 图像识别测试工具原理解读 & 最佳实践 | 京东云技术团队

@logwrapdef touch(v, times=1, **kwargs):    """    Perform the touch action on the device screen    :param v: target to touch, either a ``Template`` instance or absolute coordinates (x, y)    :param times: how many touches to be performed    :param kwargs: platform specific `kwargs`, please refer to corresponding docs    :return: finial position to be clicked, e.g. (100, 100)    :platforms: Android, Windows, iOS    """    if isinstance(v, Template):        pos = loop_find(v, timeout=ST.FIND_TIMEOUT)    else:        try_log_screen()        pos = v    for _ in range(times):        G.DEVICE.touch(pos, **kwargs)        time.sleep(0.05)    delay_after_operation()    return pos
click = touch  # click is alias of t

@logwrapdef loop_find(query, timeout=ST.FIND_TIMEOUT, threshold=None, interval=0.5, intervalfunc=None):    """    Search for image template in the screen until timeout    Args:        query: image template to be found in screenshot        timeout: time interval how long to look for the image template        threshold: default is None        interval: sleep interval before next attempt to find the image template        intervalfunc: function that is executed after unsuccessful attempt to find the image template    Raises:        TargetNotFoundError: when image template is not found in screenshot    Returns:        TargetNotFoundError if image template not found, otherwise returns the position where the image template has        been found in screenshot    """    G.LOGGING.info("Try finding: %s", query)    start_time = time.time()    while True:        screen = G.DEVICE.snapshot(filename=None, quality=ST.SNAPSHOT_QUALITY)        if screen is None:            G.LOGGING.warning("Screen is None, may be locked")        else:            if threshold:                query.threshold = threshold            match_pos = query.match_in(screen)            if match_pos:                try_log_screen(screen)                return match_pos        if intervalfunc is not None:            intervalfunc()        # 超时则raise，未超时则进行下次循环:        if (time.time() - start_time) > timeout:            try_log_screen(screen)            raise TargetNotFoundError('Picture %s not found in screen' % query)        else:            t

def match_in(self, screen):    match_result = self._cv_match(screen)    G.LOGGING.debug("match result: %s", match_result)    if not match_result:        return None    focus_pos = TargetPos().getXY(match_result, self.target_pos)    return focus_pos

@logwrapdef _cv_match(self, screen):    # in case image file not exist in current directory:    ori_image = self._imread()    image = self._resize_image(ori_image, screen, ST.RESIZE_METHOD)    ret = None    for method in ST.CVSTRATEGY:        # get function definition and execute:        func = MATCHING_METHODS.get(method, None)        if func is None:            raise InvalidMatchingMethodError("Undefined method in CVSTRATEGY: '%s', try 'kaze'/'brisk'/'akaze'/'orb'/'surf'/'sift'/'brief' instead." % method)        else:            if method in ["mstpl", "gmstpl"]:                ret = self._try_match(func, ori_image, screen, threshold=self.threshold, rgb=self.rgb, record_pos=self.record_pos,                                        resolution=self.resolution, scale_max=self.scale_max, scale_step=self.scale_step)            else:                ret = self._try_match(func, image, screen, threshold=self.threshold, rgb=self.rgb)        if ret:            break    return ret

CVSTRATEGY = ["mstpl", "tpl", "surf", "brisk"]if LooseVersion(cv2.__version__) > LooseVersion('3.4.2'):    CVSTRATEGY = ["mstpl", "tpl", "sift", "brisk"]

if method in ["mstpl", "gmstpl"]:    ret = self._try_match(func, ori_image, screen, threshold=self.threshold, rgb=self.rgb, record_pos=self.record_pos,                            resolution=self.resolution, scale_max=self.scale_max, scale_step=self.scale_step)else:    ret = self._try_match(func, image, screen, threshold=self.threshold, rgb=self.rgb)

@staticmethoddef _try_match(func, *args, **kwargs):    G.LOGGING.debug("try match with %s" % func.__name__)    try:        ret = func(*args, **kwargs).find_best_result()    except aircv.NoModuleError as err:        G.LOGGING.warning("'surf'/'sift'/'brief' is in opencv-contrib module. You can use 'tpl'/'kaze'/'brisk'/'akaze'/'orb' in CVSTRATEGY, or reinstall opencv with the contrib module.")        return None    except aircv.BaseError as err:        G.LOGGING.debug(repr(err))        return None    else:        return ret

@print_run_timedef find_best_result(self):    """基于kaze进行图像识别，只筛选出最优区域."""    """函数功能：找到最优结果."""    # 第一步：校验图像输入    check_source_larger_than_search(self.im_source, self.im_search)    # 第二步：计算模板匹配的结果矩阵res    res = self._get_template_result_matrix()    # 第三步：依次获取匹配结果    min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res)    h, w = self.im_search.shape[:2]    # 求取可信度:    confidence = self._get_confidence_from_matrix(max_loc, max_val, w, h)    # 求取识别位置: 目标中心 + 目标区域:    middle_point, rectangle = self._get_target_rectangle(max_loc, w, h)    best_match = generate_result(middle_point, rectangle, confidence)    LOGGING.debug("[%s] threshold=%s, result=%s" % (self.METHOD_NAME, self.threshold, best_match))    return best_match if confidence >= self.threshold else Non

def _get_template_result_matrix(self):    """求取模板匹配的结果矩阵."""    # 灰度识别: cv2.matchTemplate( )只能处理灰度图片参数    s_gray, i_gray = img_mat_rgb_2_gray(self.im_search), img_mat_rgb_2_gray(self.im_source)    return cv2.matchTemplate(i_gray, s_gray, cv2.TM_CCOEFF_NORMED)