Xtykutl

I'm trying to build an android accessibility service that uses OpenCV hand waves to do basic controls of other apps (like scrolling). I'm currently using NewHandWave to recognize hand waves. This has been working great except for one big problem; the JavaCameraView is creating a very large black rectangle over most of the screen. It is recognizing the hand waves and everything underneath the rectangle can still be interacted with.

Relevant code from the service java file:

package com.example.android.handgestureservice;



import android.accessibilityservice.AccessibilityService;

import android.accessibilityservice.GestureDescription;

import android.graphics.Path;

import android.graphics.PixelFormat;

import android.view.Gravity;

import android.view.WindowManager;

import android.view.accessibility.AccessibilityEvent;

import android.view.accessibility.AccessibilityNodeInfo;



import edu.washington.cs.touchfreelibrary.sensors.CameraGestureSensor;

import edu.washington.cs.touchfreelibrary.sensors.ClickSensor;

import edu.washington.cs.touchfreelibrary.utilities.LocalOpenCV;



import java.util.ArrayDeque;

import java.util.Deque;



public class HandGestureService extends AccessibilityService implements CameraGestureSensor.Listener, ClickSensor.Listener{





@Override

public void onGestureUp(CameraGestureSensor caller, long gestureLength) {

    AccessibilityNodeInfo scrollable = findScrollableNode(getRootInActiveWindow());

    if (scrollable != null) {

        scrollable.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_BACKWARD.getId());

    }





}



@Override

public void onGestureDown(CameraGestureSensor caller, long gestureLength) {

    AccessibilityNodeInfo scrollable = findScrollableNode(getRootInActiveWindow());

    if (scrollable != null) {

        scrollable.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_FORWARD.getId());

    }





}



@Override

public void onGestureLeft(CameraGestureSensor caller, long gestureLength) {

    AccessibilityNodeInfo scrollable = findScrollableNode(getRootInActiveWindow());



    Path swipePath = new Path();

    swipePath.moveTo(100, 1000);

    swipePath.lineTo(1000, 1000);

    GestureDescription.Builder gestureBuilder = new GestureDescription.Builder();

    gestureBuilder.addStroke(new GestureDescription.StrokeDescription(swipePath, 0, 500));

    dispatchGesture(gestureBuilder.build(), null, null);



}



@Override

public void onGestureRight(CameraGestureSensor caller, long gestureLength) {

    AccessibilityNodeInfo scrollable = findScrollableNode(getRootInActiveWindow());





    Path swipePath = new Path();

    swipePath.moveTo(1000, 1000);

    swipePath.lineTo(100, 1000);

    GestureDescription.Builder gestureBuilder = new GestureDescription.Builder();

    gestureBuilder.addStroke(new GestureDescription.StrokeDescription(swipePath, 0, 500));

    dispatchGesture(gestureBuilder.build(), null, null);



}



@Override

public void onSensorClick(ClickSensor caller) {



}





// Figures out if scrolling is possible.

private AccessibilityNodeInfo findScrollableNode(AccessibilityNodeInfo root) {

    Deque<AccessibilityNodeInfo> deque = new ArrayDeque<>();

    deque.add(root);

    while (!deque.isEmpty()) {

        AccessibilityNodeInfo node = deque.removeFirst();

        if (node.getActionList().contains(AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_FORWARD)) {

            return node;

        }

        for (int i = 0; i < node.getChildCount(); i++) {

            deque.addLast(node.getChild(i));

        }

    }

    return null;

}







@Override

protected void onServiceConnected() {



    // Load OpenCV

    LocalOpenCV loader = new LocalOpenCV(this,this,this);



    // Get window manager to be able to bind layouts to service

    WindowManager wm = (WindowManager) getSystemService(WINDOW_SERVICE);



    // Set up layout parameters

    WindowManager.LayoutParams lp = new WindowManager.LayoutParams();

    lp.type = WindowManager.LayoutParams.TYPE_ACCESSIBILITY_OVERLAY;

    lp.format = PixelFormat.TRANSPARENT;

    lp.flags |= WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE;

    lp.flags |= WindowManager.LayoutParams.FLAG_NOT_TOUCHABLE;

    lp.flags |= WindowManager.LayoutParams.FLAG_HARDWARE_ACCELERATED;

    lp.width = WindowManager.LayoutParams.WRAP_CONTENT;

    lp.height = WindowManager.LayoutParams.WRAP_CONTENT;

    lp.gravity = Gravity.TOP;



    // Add view to service

    wm.addView(loader.CameraLayout,lp);



}



@Override

public void onAccessibilityEvent(AccessibilityEvent event) {



}



@Override

public void onInterrupt() {



}



}

From LocalOpenCV.java from NewHandWave

// Added so that the service can bind the layout

public RelativeLayout CameraLayout = null;





// Modified to accept any context instead of Activity contexts

public LocalOpenCV(Context context, CameraGestureSensor.Listener gestureListener, ClickSensor.Listener clickListener) {

    this.context = context;

    this.gestureListener = gestureListener;

    this.clickListener = clickListener;

    doLoad(context, gestureListener, clickListener);

}



// Modified to accept any context instead of just Activity contexts

public void doLoad(Context context, CameraGestureSensor.Listener listeners, ClickSensor.Listener clickListener) {

    this.CameraLayout = new BackendGestureCamera().getJavaCameraViewWrappedInRelativeLayout(context);

    this.mCamera = (JavaCameraView) this.CameraLayout.findViewById(R.id.camera);

    makeGenericLoaderCallback();

    loadOpenCV(context);

}

From BackendGestureDetector.java from NewHandWave

public class BackendGestureCamera {



// Modified to accept any context instead of just Activity

public RelativeLayout getJavaCameraViewWrappedInRelativeLayout(Context context) {

    LayoutInflater inflater = LayoutInflater.from(context);

    RelativeLayout layout = (RelativeLayout) inflater.inflate(R.layout.item_java_camera_view, null);

    WindowManager.LayoutParams params = new WindowManager.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);

    layout.setLayoutParams(params);

    return layout;

    }

}

So far I've determined it is the CameraLayout generated by getJavaCameraViewWrappedInRelativeLayout creating the large black rectangle. I've tried setting the alpha to 0 (which didn't do anything) and changing the visibility to INVISIBLE or GONE (which gets rid of the rectangle but disables the camera).

TLDR: Looking for a way of making a layout invisible without disabling the OpenCV JavaCameraView.