1.
电脑操作手机使用 WebDriverAgent.
从 https://github.com/facebook/WebDriverAgent clone 后 ,按照 github 上的说明,可以正确生成 xcode 工程.
WebDriverAgent的 xcode 工程依赖 Carthage 和 node 两个库,可以用 brew 安装
2.
WebDriverAgent xcode 工程设置成功后,打开 WebDriverAgent.xcodeproj
把里面 的 WebDriverAgentRunner target 用 Product -> Test 的方式 安装到手机上,并运行 app
过程中可能需要 设置自己的开发账号
工程部署好了之后,可以通过 xcode 命令行 自动 build WebDriverAgent 到 手机上
xcodebuild -project WebDriverAgent.xcodeproj -scheme WebDriverAgentRunner -destination "id=$(idevice_id -l)" test
3.
WebDriverAgentRunner 部署到 手机上成功后 ,用 brew 安装 usbuxd库,因为要使用 iproxy 命令远程操作手机
4.
使用 iproxy 8100 8100 把 手机的 8100 端口,映射到电脑的 8100 端口上。
然后访问 电脑 http://localhost:8100 等同于访问手机的 8100 端口 。浏览器上 打开 http://localhost:8100 如果有信息的话,电脑远程操作 手机的 准备工作就做好了。
5.
下载安装 python 的 atx 库。pip install atx
atx 给 python 提供了 各种直接操作手机的方法。
示例:
d = atx.connect('http://localhost:8100', platform='ios')
image = d.screenshot();
image.save("./testscreen.png");
d.click(200,200);
d.swipe(100,100,200,200,duration);
6.
python 下载cv2 库,使用 opencv 的功能
cv2 用于 分析截屏图片
使用示例
读取:
img = cv2.imread("./osizejump.jpg");
显示:
cv2.namedWindow("debug");
cv2.imshow("debug",img);
cv2.waitKey(0);
绘制(比如在图片上做标记):
cv2.namedWindow("debug");
cv2.rectangle(img,topLeft,bottomRight,(0,0,255),2);
cv2.imshow("debug",img);
cv2.waitKey(0);
图像匹配:
res = cv2.matchTemplate(screenImg,jumperTemp,cv2.TM_CCOEFF_NORMED);
min_val,max_val,min_loc,max_loc = cv2.minMaxLoc(res);
生成canny 图片(用于做边缘检测):
img = cv2.GaussianBlur(screenImg,(3,3),0); #500多分 的时候 都没用 高斯模糊
canny = cv2.Canny(img,35,100);
保存图片:
cv2.imwrite("./dest/temcanny.jpg",canny);
7.
自动跳跃思路:
(1) 使用 atx 远程操作手机 截屏,保存本地文件
(2) 使用 cv2 读取屏幕图片文件
(3) cv2.matchTemplate() 匹配 棋子,用于计算棋子位置
(4) cv2 生成 canny 图片用于做边缘检测
(5) 根据边缘检测图片,计算目标坐标点
(6) 根据棋子位置,和目标坐标点 ,计算距离,设置 长按时间
(7) 使用 atx swipe() 长按屏幕,完成自动跳跃
8.
代码没什么参考价值。写在这里做备忘
# -*- coding:utf-8 -*- import atx import cv2 import numpy as np import math import time d = None; d = atx.connect("http://localhost:8100"); screen = None; screenImg = None; jumperTemp = None; jumperW = None; jumperH = None; targetTemp = None; targetW = None; targetH = None; def debugImage(img): cv2.namedWindow("debug"); cv2.imshow("debug",img); cv2.waitKey(0); def debugRect(img,topLeft,bottomRight): cv2.namedWindow("debug"); cv2.rectangle(img,topLeft,bottomRight,(0,0,255),2); cv2.imshow("debug",img); cv2.waitKey(0); def debugPoint(img,x,y): cv2.namedWindow("debug"); cv2.circle(img,(int(x),int(y)),10,(0,0,255),-1); cv2.imshow("debug",img); cv2.waitKey(0); def initConst(): global jumperTemp global jumperW global jumperH jumperTemp = cv2.imread("./osizejump.jpg"); jumperW,jumperH = jumperTemp.shape[1],jumperTemp.shape[0]; global targetTemp global targetW global targetH targetTemp = cv2.imread("./osizetarget.jpg"); targetW,targetH = targetTemp.shape[1],targetTemp.shape[0]; ''' src pos begin ''' def genScreenshot(): global screen global screenImg screen = d.screenshot(); screen.save("./dest/screen.jpg"); screenImg = cv2.imread("./dest/screen.jpg"); def matchJumper(): global screenImg; global jumperTemp; global jumperW global jumperH res = cv2.matchTemplate(screenImg,jumperTemp,cv2.TM_CCOEFF_NORMED); # print(res); # debugImage(res); min_val,max_val,min_loc,max_loc = cv2.minMaxLoc(res); top_left = max_loc; bottom_right = (top_left[0] + jumperW,top_left[1] + jumperH); # debugRect(screenImg,top_left,bottom_right); x = top_left[0] + jumperW/2; y = top_left[1] + jumperH * 0.85; # debugPoint(screenImg,x,y); return x,y,min_loc,max_loc; def getSrcPos(): return matchJumper(); ''' src pos end ''' ''' dest pos begin ''' def get_center(img_canny, ): CHECK_HEIGHT = 100; START_CHECK_ROW = 400 # debugImage(img_canny); theRow = np.nonzero([max(row) for row in img_canny[START_CHECK_ROW:]]) # print("miao1 row:" + str(row)); # print("miao1") # print(row); # print(pt); # print("miao1 end"); # print("miao2") # print(theRow) # print("miao2 end") y_top = theRow[0][0] + START_CHECK_ROW # y_top = theRow[0][0] #@miao @test print("y_top:" + str(y_top)) x_top = int(np.mean(np.nonzero(img_canny[y_top]))) print("x_top:" + str(x_top)); H = img_canny.shape[0]; y_bottom = y_top + CHECK_HEIGHT print("y_bottom:" + str(y_bottom)); for row in range(y_bottom, H): # print("check row:" + str(row)); if img_canny[row, x_top] != 0: y_bottom = row break x_center, y_center = x_top, (y_top + y_bottom) // 2 return img_canny, x_center, y_center def detectEdge(): global screenImg; img = cv2.GaussianBlur(screenImg,(3,3),0); #500多分 的时候 都没用 高斯模糊 img = screenImg; # debugImage(img); # canny = cv2.Canny(img,50,150); # canny = cv2.Canny(img,10,50); canny = cv2.Canny(img,35,100); # debugImage(canny); return canny; def detectTarget(): global screenImg; global targetTemp global targetW global targetH res = cv2.matchTemplate(screenImg,targetTemp,cv2.TM_CCOEFF_NORMED); # print(res); # debugImage(res); min_val,max_val,min_loc,max_loc = cv2.minMaxLoc(res); print("------- max_val:" + str(max_val)); if max_val >= 0.8: print(" match target !"); top_left = max_loc; bottom_right = (top_left[0] + targetW,top_left[1] + targetH); # debugRect(screenImg,top_left,bottom_right); x = top_left[0] + targetW/2; y = top_left[1] + targetH/2; # print(screenImg) # debugImage(screenImg); # print("x:" + str(x) + ",y:" + str(y)) # debugPoint(screenImg,x,y); # print("miaomiao"); return True,x,y; print("unmatch target"); return False,0,0 def getDestPos(jumperMinLoc,jumperMaxLoc): global jumperW global jumperH global screenImg canny = detectEdge(); # remove jumper in canny for k in range(jumperMaxLoc[1] - 10, jumperMaxLoc[1] + jumperH): for b in range(jumperMaxLoc[0] - 10, jumperMaxLoc[0] + jumperW): canny[k][b] = 0 # debugImage(canny); cv2.imwrite("./dest/temcanny.jpg",canny); # print(canny) x = 0 y = 0 # if detectTarget(): bResult,x,y = detectTarget(); if bResult: print("got target spot"); return x,y; else: canny,x,y = get_center(canny) print("find next center"); # print(x) # print(y); # debugPoint(screenImg,x,y); return x,y ''' dest pos end ''' def playStep(): global d global screenImg genScreenshot(); x1,y1,minLoc,maxLoc = getSrcPos(); # debugPoint(screenImg,x1,y1); x2,y2 = getDestPos(minLoc,maxLoc); # debugPoint(screenImg,x2,y2); print("src:("+str(x1) + ","+str(y1) + ")") print("dst:("+str(x1) + ","+str(y2) + ")") cv2.circle(screenImg,(int(x1),int(y1)),10,(0,0,255),-1); cv2.circle(screenImg,(int(x2),int(y2)),10,(255,0,0),-1); cv2.imwrite("./dest/fenxi.jpg",screenImg); distance = math.pow((x2 - x1),2) + math.pow((y2 - y1),2) distance = math.sqrt(distance) print("distance:"+str(distance)); # duration = distance / 1047.1 #score 431 # duration = distance / 1046.8 # score 554 duration = distance / 1046.5 print("duration:"+str(duration)); if distance < 429 and distance > 300: duration = duration - 0.04 print("fix 0 ..") elif distance <= 300: print("fix 1 .."); duration = duration - 0.09; ''' FAIL !!! @miao @todo dst:(519,982) distance:191.095689381 duration:0.182552244346 fix 1 .. fix 2... final duration:0.1 ^CTraceback (most recent call la unmatch target y_top:932 x_top:654 y_bottom:1032 find next center src:(519,1117.25) dst:(519,982) distance:191.095689381 duration:0.182552244346 fix 1 .. fix 2... final duration:0.1 ^CTraceback (most recent call last): File "dest2.py", line 258, in <module> playStep(); ''' if duration < 0.07: duration = 0.07; print("fix 2..."); print("final duration:" + str(duration)); d.swipe(100,100,200,200,duration); initConst(); while True: time.sleep(0.9) playStep(); # playStep(); # test case # genScreenshot(); # screenImg = cv2.imread("./dest/hastarget.jpg"); # detectTarget(); # matchJumper();