libobs: UI: Fix rotated line scale

The line drawing functions previously assumed the upper-left 3x3 for
box_transform only held scale. The matrix can also hold rotation, so
pass in scale separately.

Fixes https://obsproject.com/mantis/view.php?id=1442

UI/window-basic-preview.cpp

@@ -1235,21 +1235,19 @@ static void DrawSquareAtPos(float x, float y)
 	gs_matrix_pop();
 }
 
-static void DrawLine(float x1, float y1, float x2, float y2, float thickness)
+static void DrawLine(float x1, float y1, float x2, float y2, float thickness,
+		vec2 scale)
 {
-	struct matrix4 matrix;
-	gs_matrix_get(&matrix);
-
 	float ySide = (y1 == y2) ? (y1 < 0.5f ? 1.0f : -1.0f) : 0.0f;
 	float xSide = (x1 == x2) ? (x1 < 0.5f ? 1.0f : -1.0f) : 0.0f;
 
 	gs_render_start(true);
 
 	gs_vertex2f(x1, y1);
-	gs_vertex2f(x1 + (xSide * (thickness / matrix.x.x)),
-		y1 + (ySide * (thickness / matrix.y.y)));
-	gs_vertex2f(x2 + (xSide * (thickness / matrix.x.x)),
-		y2 + (ySide * (thickness / matrix.y.y)));
+	gs_vertex2f(x1 + (xSide * (thickness / scale.x)),
+		y1 + (ySide * (thickness / scale.y)));
+	gs_vertex2f(x2 + (xSide * (thickness / scale.x)),
+		y2 + (ySide * (thickness / scale.y)));
 	gs_vertex2f(x2, y2);
 	gs_vertex2f(x1, y1);
 
@@ -1260,31 +1258,28 @@ static void DrawLine(float x1, float y1, float x2, float y2, float thickness)
 	gs_vertexbuffer_destroy(line);
 }
 
-static void DrawRect(float thickness)
+static void DrawRect(float thickness, vec2 scale)
 {
-	struct matrix4 matrix;
-	gs_matrix_get(&matrix);
-
 	gs_render_start(true);
 
 	gs_vertex2f(0.0f, 0.0f);
-	gs_vertex2f(0.0f + (thickness / matrix.x.x), 0.0f);
-	gs_vertex2f(0.0f + (thickness / matrix.x.x), 1.0f);
+	gs_vertex2f(0.0f + (thickness / scale.x), 0.0f);
+	gs_vertex2f(0.0f + (thickness / scale.x), 1.0f);
 	gs_vertex2f(0.0f, 1.0f);
 	gs_vertex2f(0.0f, 0.0f);
 	gs_vertex2f(0.0f, 1.0f);
-	gs_vertex2f(0.0f, 1.0f - (thickness / matrix.y.y));
-	gs_vertex2f(1.0f, 1.0f - (thickness / matrix.y.y));
+	gs_vertex2f(0.0f, 1.0f - (thickness / scale.y));
+	gs_vertex2f(1.0f, 1.0f - (thickness / scale.y));
 	gs_vertex2f(1.0f, 1.0f);
 	gs_vertex2f(0.0f, 1.0f);
 	gs_vertex2f(1.0f, 1.0f);
-	gs_vertex2f(1.0f - (thickness / matrix.x.x), 1.0f);
-	gs_vertex2f(1.0f - (thickness / matrix.x.x), 0.0f);
+	gs_vertex2f(1.0f - (thickness / scale.x), 1.0f);
+	gs_vertex2f(1.0f - (thickness / scale.x), 0.0f);
 	gs_vertex2f(1.0f, 0.0f);
 	gs_vertex2f(1.0f, 1.0f);
 	gs_vertex2f(1.0f, 0.0f);
-	gs_vertex2f(1.0f, 0.0f + (thickness / matrix.y.y));
-	gs_vertex2f(0.0f, 0.0f + (thickness / matrix.y.y));
+	gs_vertex2f(1.0f, 0.0f + (thickness / scale.y));
+	gs_vertex2f(0.0f, 0.0f + (thickness / scale.y));
 	gs_vertex2f(0.0f, 0.0f);
 	gs_vertex2f(1.0f, 0.0f);
 
@@ -1456,6 +1451,13 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t *scene,
 
 	GS_DEBUG_MARKER_BEGIN(GS_DEBUG_COLOR_DEFAULT, "DrawSelectedItem");
 
+	matrix4 curTransform;
+	vec2 boxScale;
+	gs_matrix_get(&curTransform);
+	obs_sceneitem_get_box_scale(item, &boxScale);
+	boxScale.x *= curTransform.x.x;
+	boxScale.y *= curTransform.y.y;
+
 	obs_transform_info info;
 	obs_sceneitem_get_info(item, &info);
 
@@ -1474,7 +1476,7 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t *scene,
 			gs_effect_set_vec4(colParam, &blue); \
 		else if (crop.side > 0) \
 			gs_effect_set_vec4(colParam, &green); \
-		DrawLine(x1, y1, x2, y2, HANDLE_RADIUS / 2); \
+		DrawLine(x1, y1, x2, y2, HANDLE_RADIUS / 2, boxScale); \
 		gs_effect_set_vec4(colParam, &red);
 
 		DRAW_SIDE(left,   0.0f, 0.0f, 0.0f, 1.0f);
@@ -1485,9 +1487,9 @@ bool OBSBasicPreview::DrawSelectedItem(obs_scene_t *scene,
 	} else {
 		if (!selected) {
 			gs_effect_set_vec4(colParam, &blue);
-			DrawRect(HANDLE_RADIUS / 2);
+			DrawRect(HANDLE_RADIUS / 2, boxScale);
 		} else {
-			DrawRect(HANDLE_RADIUS / 2);
+			DrawRect(HANDLE_RADIUS / 2, boxScale);
 		}
 	}
 

libobs/obs-scene.c

@@ -395,6 +395,8 @@ static void update_item_transform(struct obs_scene_item *item, bool update_tex)
 		scale.y = (float)height * item->scale.y;
 	}
 
+	item->box_scale = scale;
+
 	add_alignment(&base_origin, item->align, (int)scale.x, (int)scale.y);
 
 	matrix4_identity(&item->box_transform);
@@ -1260,6 +1262,7 @@ static inline void duplicate_item_data(struct obs_scene_item *dst,
 	dst->output_scale = src->output_scale;
 	dst->scale_filter = src->scale_filter;
 	dst->box_transform = src->box_transform;
+	dst->box_scale = src->box_scale;
 	dst->draw_transform = src->draw_transform;
 	dst->bounds_type = src->bounds_type;
 	dst->bounds_align = src->bounds_align;
@@ -2047,6 +2050,13 @@ void obs_sceneitem_get_box_transform(const obs_sceneitem_t *item,
 		matrix4_copy(transform, &item->box_transform);
 }
 
+void obs_sceneitem_get_box_scale(const obs_sceneitem_t *item,
+		struct vec2 *scale)
+{
+	if (item)
+		*scale = item->box_scale;
+}
+
 bool obs_sceneitem_visible(const obs_sceneitem_t *item)
 {
 	return item ? item->user_visible : false;

libobs/obs-scene.h

@@ -65,6 +65,7 @@ struct obs_scene_item {
 	enum obs_scale_type   scale_filter;
 
 	struct matrix4        box_transform;
+	struct vec2           box_scale;
 	struct matrix4        draw_transform;
 
 	enum obs_bounds_type  bounds_type;

libobs/obs.h

@@ -1453,6 +1453,8 @@ EXPORT void obs_sceneitem_get_draw_transform(const obs_sceneitem_t *item,
 		struct matrix4 *transform);
 EXPORT void obs_sceneitem_get_box_transform(const obs_sceneitem_t *item,
 		struct matrix4 *transform);
+EXPORT void obs_sceneitem_get_box_scale(const obs_sceneitem_t *item,
+		struct vec2 *scale);
 
 EXPORT bool obs_sceneitem_visible(const obs_sceneitem_t *item);
 EXPORT bool obs_sceneitem_set_visible(obs_sceneitem_t *item, bool visible);