[media] mem2mem_testdev: unlock and return error code properly

We recently added locking to this function, but there was an error path
which accidentally returned holding a lock.  Also we returned zero on
failure on some paths instead of the error code.
Signed-off-by: NDan Carpenter <dan.carpenter@oracle.com>
Signed-off-by: NMauro Carvalho Chehab <mchehab@redhat.com>

drivers/media/platform/mem2mem_testdev.c

@@ -911,10 +911,9 @@ static int m2mtest_open(struct file *file)
 	v4l2_ctrl_new_custom(hdl, &m2mtest_ctrl_trans_time_msec, NULL);
 	v4l2_ctrl_new_custom(hdl, &m2mtest_ctrl_trans_num_bufs, NULL);
 	if (hdl->error) {
-		int err = hdl->error;
-
+		rc = hdl->error;
 		v4l2_ctrl_handler_free(hdl);
-		return err;
+		goto open_unlock;
 	}
 	ctx->fh.ctrl_handler = hdl;
 	v4l2_ctrl_handler_setup(hdl);
@@ -946,7 +945,7 @@ static int m2mtest_open(struct file *file)
 
 open_unlock:
 	mutex_unlock(&dev->dev_mutex);
-	return 0;
+	return rc;
 }
 
 static int m2mtest_release(struct file *file)