Added stencil masking

include/sway/desktop/fx_renderer.h

@@ -47,6 +47,8 @@ struct fx_renderer {
 
 	float projection[9];
 
+	GLuint stencil_buffer_id;
+
 	struct {
 		bool OES_egl_image_external;
 	} exts;

sway/desktop/fx_renderer.c

@@ -285,6 +285,17 @@ error:
 }
 
 void fx_renderer_begin(struct fx_renderer *renderer, uint32_t width, uint32_t height) {
+	// Create and render the stencil buffer
+	if (renderer->stencil_buffer_id == 0) {
+		glGenRenderbuffers(1, &renderer->stencil_buffer_id);
+		glBindRenderbuffer(GL_RENDERBUFFER, renderer->stencil_buffer_id);
+		glRenderbufferStorage(GL_RENDERBUFFER, GL_STENCIL_INDEX8, width, height);
+		// TODO: Needed?
+		int status = glCheckFramebufferStatus(GL_FRAMEBUFFER);
+		assert(status == GL_FRAMEBUFFER_COMPLETE);
+	}
+	glFramebufferRenderbuffer(GL_FRAMEBUFFER, GL_STENCIL_ATTACHMENT, GL_RENDERBUFFER, renderer->stencil_buffer_id);
+
 	glViewport(0, 0, width, height);
 
 	// refresh projection matrix
@@ -300,7 +311,8 @@ void fx_renderer_end() {
 
 void fx_renderer_clear(const float color[static 4]) {
 	glClearColor(color[0], color[1], color[2], color[3]);
-		glClear(GL_COLOR_BUFFER_BIT);
+	glClearStencil(0);
+	glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 }
 
 void fx_renderer_scissor(struct wlr_box *box) {
@@ -606,4 +618,6 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo
 	glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
 
 	glDisableVertexAttribArray(renderer->shaders.box_shadow.pos_attrib);
+
+	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 }

sway/desktop/render.c

@@ -341,6 +341,11 @@ void render_box_shadow(struct sway_output *output, pixman_region32_t *output_dam
 	box.x -= output->lx * wlr_output->scale;
 	box.y -= output->ly * wlr_output->scale;
 
+	struct wlr_box inner_box;
+	memcpy(&inner_box, &box, sizeof(struct wlr_box));
+	// NOTE: Alpha needs to be set to 1.0 to be able to discard any "empty" pixels
+	const float col[4] = {0.0, 0.0, 0.0, 1.0};
+
 	box.x -= blur_sigma;
 	box.y -= blur_sigma;
 	box.width += 2 * blur_sigma;
@@ -350,31 +355,43 @@ void render_box_shadow(struct sway_output *output, pixman_region32_t *output_dam
 	pixman_region32_init(&damage);
 	pixman_region32_union_rect(&damage, &damage, box.x, box.y,
 		box.width, box.height);
-
-	pixman_region32_t inner_damage;
-	pixman_region32_init(&inner_damage);
-	pixman_region32_union_rect(&inner_damage, &inner_damage,
-		box.x + blur_sigma + corner_radius,
-		box.y + blur_sigma + corner_radius,
-		box.width - (blur_sigma + corner_radius) * 2.0,
-		box.height - (blur_sigma + corner_radius) * 2.0);
-
-	pixman_region32_subtract(&damage, &damage, &inner_damage);
-	pixman_region32_fini(&inner_damage);
 	pixman_region32_intersect(&damage, &damage, output_damage);
 	bool damaged = pixman_region32_not_empty(&damage);
 	if (!damaged) {
 		goto damage_finish;
 	}
 
+	// Init stencil work
+	glEnable(GL_STENCIL_TEST);
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+
 	int nrects;
 	pixman_box32_t *rects = pixman_region32_rectangles(&damage, &nrects);
 	for (int i = 0; i < nrects; ++i) {
 		scissor_output(wlr_output, &rects[i]);
-		fx_render_box_shadow(renderer, &box, color, wlr_output->transform_matrix,
-				corner_radius, blur_sigma);
+
+		// Use a rounded rect as a mask for the box shadow
+		glStencilFunc(GL_ALWAYS, 1, 0xFF);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+
+		fx_render_rounded_rect(renderer, &inner_box, col,
+				wlr_output->transform_matrix, corner_radius, ALL);
+
+		glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+
+		fx_render_box_shadow(renderer, &box, color,
+				wlr_output->transform_matrix, corner_radius, blur_sigma);
 	}
 
+	// cleanup
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glDisable(GL_STENCIL_TEST);
+
 damage_finish:
 	pixman_region32_fini(&damage);
 }

sway/desktop/shaders/quad_round.frag

@@ -12,4 +12,7 @@ void main() {
     float distance = min(max(q.x,q.y),0.0) + length(max(q,0.0)) - radius;
     float smoothedAlpha = 1.0 - smoothstep(-1.0, 1.0, distance);
     gl_FragColor = mix(vec4(0), v_color, smoothedAlpha);
+    if (gl_FragColor.a == 0.0) {
+        discard;
+    }
 }