Using SIFT algo to automation Android APP

sift.py

#!/usr/bin/python
# coding=utf-8
from __future__ import print_function
import cv2
import scipy as sp
import os
import subprocess
import time

globalStartupInfo = subprocess.STARTUPINFO()
globalStartupInfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
adbdir = "C:/Android/adb.exe"

picAlarm = "alarm.png"             # 右上角警告
picBlock = "cheat_block.png"      # 欺诈拦截
picBroadcast = "broadcast.png"    # 安全播报
picClean = "clean.jpg"             # 清理加速
picFixNow = "fix_now.png"          # 立即修复
picKillVirus = "kill_virus.png"   # 欺诈拦截
picMainPage = "main_page.png"      # 卫士主页
picSoftwareMgr= "software_mgr.png" # 软件管理
picMe = "me.png"                     # 我
picToolbox = "toolbox.png"          # 工具箱


def dur( op=None, clock=[time.time()] ):
  if op != None:
    duration = time.time() - clock[0]
    print('%s finished. Duration %.6f seconds.' % (op, duration))
  clock[0] = time.time()

def runCmd(cmd):
    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.getcwd(), shell=False,
                         startupinfo=globalStartupInfo)
    p.wait()
    re = p.stdout.read().decode()
    return re

def runAdbCmd(option, command):
    adbCmd = adbdir + " " + option + " " + command
    print("Issue ADB command: " + adbCmd)
    return runCmd(adbCmd)


def captureScreenshot():
    curdir=os.getcwd()
    picdir=curdir+"/pic"
    adbdir="C:/Android"
    localPath = ""  # where picutre will be stored on PC

    # 连接的手机列表
    mobiles = []
    cmd = [ adbdir + '/adb.exe', 'devices' ]
    mobilelist=runCmd(cmd)
    mobilelist=mobilelist.split('\r\n')[1:]

    # print(mobilelist)
    for x in mobilelist:
        if x:
            mobiles.append(x)
    if mobiles:
        print(mobiles)
    else:
        print(['no devices\t no devices'])

    # 取第一个手机的序列号
    phoneSN = '';
    if mobiles:
        # 取第一个手机设备
        device=mobiles[0].split('\t')
        phoneSN=device[0]
        print(device)

    # 有手机连接上就截图
    if phoneSN:
        # 保存到本地电脑的图片路径
        timestamp = time.strftime('%Y%m%d%H%M%S', time.localtime(time.time()))
        localPath = picdir + "/" + timestamp + '.png'
        print("Local path: ", localPath)
        sdcardpath = '/sdcard/screenshot-'+timestamp+'.png'
        if os.path.exists(localPath):
            os.remove(localPath)
        print('it is screenshoting to mobile.....')
        jtcmd=adbdir +'/adb.exe   -s '+ phoneSN + ' shell /system/bin/screencap -p '+ sdcardpath
        # print(jtcmd)
        result=runCmd(jtcmd)
        print('it is screenshot success.....')
        # print(result)
        print('it is moving screenshot to pc.....')
        jtcmd=adbdir + '/adb.exe  -s  ' + phoneSN + ' pull ' + sdcardpath + ' ' + localPath
        # print(jtcmd)
        result=runCmd(jtcmd)
        # print(result)
        # Delete picture on SD card
        jtcmd = adbdir + '/adb.exe   -s ' + phoneSN + ' shell rm  ' + sdcardpath
        # print(jtcmd)
        result=runCmd(jtcmd)
        print(result)
        print('moved screenshot to pc success.....')
    else:
        print('no device!')

    return localPath

# Initialise the timing clock
dur()

picClean = "./pic/clean.jpg"
picAlarm = "./pic/alarm.png"
# pic_screen = "./pic/screen.png"
picScreen = captureScreenshot()
picTarget = "./pic/" + picMe

print("Train image: " + picScreen + "\nTarget image: " + picTarget)
# print os.path.exists(pic_clean)
# print os.path.exists(pic_screen)

img1 = cv2.imread(picTarget, 0)  # queryImage
img2 = cv2.imread(picScreen, 0)  # trainImage

##  Initiate SIFT detector
sift = cv2.SIFT()

# find the keypoints and descriptors with SIFT
kp1, des1 = sift.detectAndCompute(img1, None)
kp2, des2 = sift.detectAndCompute(img2, None)

# FLANN parameters
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5)
search_params = dict(checks=50)  # or pass empty dictionary
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1, des2, k=2)

print('matched points number: ', len(matches))

# Apply ratio test
good = []


for m, n in matches:
    if m.distance < 0.7 * n.distance:
        good.append(m)

# get center of set of points
x = [kp2[p.trainIdx].pt[0] for p in good ]
y = [kp2[p.trainIdx].pt[1] for p in good ]
centroid=(sum(x) / len(good), sum(y)/len(good))
print("Centroid: " + str(centroid))

print('Good points number: ', len(good))
cmd = "input tap " + str(int(centroid[0])) + " " + str(int(centroid[1]))

runAdbCmd("shell", cmd)
dur("Done")
os._exit(1)
# ############################################
# visualization
# copy the target image to top-left, and the base image to bottom-right
h1, w1 = img1.shape[:2]
h2, w2 = img2.shape[:2]
view = sp.zeros((max(h1, h2), w1 + w2, 3), sp.uint8)
view[:h1, :w1, 0] = img1
view[:h2, w1:, 0] = img2
view[:, :, 1] = view[:, :, 0]
view[:, :, 2] = view[:, :, 0]

for m in good:
    # draw the keypoints
    # print kp2[m.trainIdx].pt[0],  kp2[m.trainIdx].pt[1], m.distance
    color = tuple([sp.random.randint(0, 255) for _ in xrange(3)])
    #print 'kp1,kp2',kp1,kp2
    cv2.line(view, (int(kp1[m.queryIdx].pt[0]), int(kp1[m.queryIdx].pt[1])),
            (int(kp2[m.trainIdx].pt[0] + w1), int(kp2[m.trainIdx].pt[1])), color)


cv2.imwrite("knn.jpg", view)
print("Task I is done\n")
# cv2.imshow("knn.jpg", view)
# cv2.waitKey()

img3 = cv2.imread(picScreen)  # trainImage
gray=cv2.cvtColor(img3, cv2.COLOR_BGR2GRAY)
kp = sift.detect(gray, None)

view = cv2.drawKeypoints(gray, kp, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
# view = cv2.drawKeypoints(gray, kp)
cv2.imwrite("key_pints.jpg", view)
print("Task II is done\n")