Use previously drawn as mask for shadow

include/sway/desktop/fx_renderer.h

@@ -113,6 +113,12 @@ void fx_renderer_clear(const float color[static 4]);
 
 void fx_renderer_scissor(struct wlr_box *box);
 
+void fx_renderer_start_stenciling(bool color_mask_write);
+
+void fx_renderer_close_stenciling(bool color_mask_write);
+
+void fx_renderer_end_stenciling();
+
 bool fx_render_subtexture_with_matrix(struct fx_renderer *renderer, struct wlr_texture *wlr_texture,
 		const struct wlr_fbox *src_box, const struct wlr_box *dst_box, const float matrix[static 9],
 		struct decoration_data deco_data);

sway/desktop/fx_renderer.c

@@ -322,6 +322,34 @@ void fx_renderer_scissor(struct wlr_box *box) {
 		}
 }
 
+void fx_renderer_start_stenciling(bool color_mask_write) {
+	glEnable(GL_STENCIL_TEST);
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+
+	glStencilFunc(GL_ALWAYS, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+	// Disable writing to color buffer
+	if (!color_mask_write) {
+		glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+	}
+}
+
+void fx_renderer_close_stenciling(bool color_mask_write) {
+	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+	// Reenable writing to color buffer
+	if (!color_mask_write) {
+		glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	}
+}
+
+void fx_renderer_end_stenciling() {
+	glClearStencil(0);
+	glClear(GL_STENCIL_BUFFER_BIT);
+	glDisable(GL_STENCIL_TEST);
+}
+
 bool fx_render_subtexture_with_matrix(struct fx_renderer *renderer, struct wlr_texture *wlr_texture,
 		const struct wlr_fbox *src_box, const struct wlr_box *dst_box, const float matrix[static 9],
 		struct decoration_data deco_data) {
@@ -591,31 +619,6 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo
 
 	wlr_matrix_transpose(gl_matrix, gl_matrix);
 
-	// Init stencil work
-	// NOTE: Alpha needs to be set to 1.0 to be able to discard any "empty" pixels
-	const float col[4] = {0.0, 0.0, 0.0, 1.0};
-	struct wlr_box inner_box;
-	memcpy(&inner_box, box, sizeof(struct wlr_box));
-	inner_box.x += blur_sigma;
-	inner_box.y += blur_sigma;
-	inner_box.width  -= blur_sigma * 2;
-	inner_box.height -= blur_sigma * 2;
-
-	glEnable(GL_STENCIL_TEST);
-	glClearStencil(0);
-	glClear(GL_STENCIL_BUFFER_BIT);
-
-	// Use a rounded rect as a mask for the box shadow
-	glStencilFunc(GL_ALWAYS, 1, 0xFF);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-
-	fx_render_rounded_rect(renderer, &inner_box, col, projection, corner_radius, ALL);
-
-	glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-
 	// blending will practically always be needed (unless we have a madman
 	// who uses opaque shadows with zero sigma), so just enable it
 	glEnable(GL_BLEND);
@@ -642,9 +645,4 @@ void fx_render_box_shadow(struct fx_renderer *renderer, const struct wlr_box *bo
 	glDisableVertexAttribArray(renderer->shaders.box_shadow.pos_attrib);
 
 	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
-
-	// cleanup
-	glClearStencil(0);
-	glClear(GL_STENCIL_BUFFER_BIT);
-	glDisable(GL_STENCIL_TEST);
 }

sway/desktop/render.c

@@ -586,17 +586,6 @@ static void render_view(struct sway_output *output, pixman_region32_t *damage,
 			}
 		}
 	}
-
-	// render shadow
-	if (con->shadow_enabled
-		&& config->shadow_blur_sigma > 0
-		&& config->shadow_color[3] > 0.0) {
-		struct wlr_box box = { state->x, state->y, state->width, state->height };
-		scale_box(&box, output_scale);
-		render_box_shadow(output, damage, &box, config->shadow_color,
-				config->shadow_blur_sigma, deco_data.corner_radius,
-				con->current.border_thickness);
-	}
 }
 
 /**
@@ -1011,6 +1000,9 @@ static void render_containers_linear(struct sway_output *output,
 				.saturation = child->saturation,
 				.has_titlebar = has_titlebar,
 			};
+
+			fx_renderer_start_stenciling(true);
+
 			render_view(output, damage, child, colors, deco_data);
 			if (has_titlebar) {
 				render_titlebar(output, damage, child, floor(state->x), floor(state->y),
@@ -1019,6 +1011,21 @@ static void render_containers_linear(struct sway_output *output,
 			} else if (state->border == B_PIXEL) {
 				render_top_border(output, damage, state, colors, deco_data.alpha, deco_data.corner_radius);
 			}
+			
+			fx_renderer_close_stenciling(true);
+			
+			// render shadow
+			if (child->shadow_enabled
+				&& config->shadow_blur_sigma > 0
+				&& config->shadow_color[3] > 0.0) {
+				struct wlr_box box = { state->x, state->y, state->width, state->height };
+				scale_box(&box, output->wlr_output->scale);
+				render_box_shadow(output, damage, &box, config->shadow_color,
+						config->shadow_blur_sigma, deco_data.corner_radius,
+						state->border_thickness);
+			}
+			
+			fx_renderer_end_stenciling();
 		} else {
 			render_container(output, damage, child,
 					parent->focused || child->current.focused);
@@ -1046,6 +1053,8 @@ static void render_containers_tabbed(struct sway_output *output,
 	struct border_colors *current_colors = &config->border_colors.unfocused;
 	int tab_width = parent->box.width / parent->children->length;
 
+	fx_renderer_start_stenciling(true);
+
 	// Render tabs
 	for (int i = 0; i < parent->children->length; ++i) {
 		struct sway_container *child = parent->children->items[i];
@@ -1111,6 +1120,26 @@ static void render_containers_tabbed(struct sway_output *output,
 		render_container(output, damage, current,
 				parent->focused || current->current.focused);
 	}
+
+	fx_renderer_close_stenciling(true);
+
+	// render shadow
+	if (current->shadow_enabled
+		&& config->shadow_blur_sigma > 0
+		&& config->shadow_color[3] > 0.0) {
+		struct wlr_box box = {
+			current->current.x,
+			current->current.y,
+			current->current.width,
+			current->current.height
+		};
+		scale_box(&box, output->wlr_output->scale);
+		render_box_shadow(output, damage, &box, config->shadow_color,
+				config->shadow_blur_sigma, current->corner_radius,
+				current->current.border_thickness);
+	}
+
+	fx_renderer_end_stenciling();
 }
 
 /**
@@ -1125,6 +1154,8 @@ static void render_containers_stacked(struct sway_output *output,
 	struct border_colors *current_colors = &config->border_colors.unfocused;
 	size_t titlebar_height = container_titlebar_height();
 
+	fx_renderer_start_stenciling(true);
+
 	// Render titles
 	for (int i = 0; i < parent->children->length; ++i) {
 		struct sway_container *child = parent->children->items[i];
@@ -1185,6 +1216,26 @@ static void render_containers_stacked(struct sway_output *output,
 		render_container(output, damage, current,
 				parent->focused || current->current.focused);
 	}
+
+	fx_renderer_close_stenciling(true);
+
+	// render shadow
+	if (current->shadow_enabled
+		&& config->shadow_blur_sigma > 0
+		&& config->shadow_color[3] > 0.0) {
+		struct wlr_box box = {
+			current->current.x,
+			current->current.y,
+			current->current.width,
+			current->current.height
+		};
+		scale_box(&box, output->wlr_output->scale);
+		render_box_shadow(output, damage, &box, config->shadow_color,
+				config->shadow_blur_sigma, current->corner_radius,
+				current->current.border_thickness);
+	}
+
+	fx_renderer_end_stenciling();
 }
 
 static void render_containers(struct sway_output *output,
@@ -1280,6 +1331,7 @@ static void render_floating_container(struct sway_output *soutput,
 			.corner_radius = con->corner_radius,
 			.has_titlebar = has_titlebar,
 		};
+		fx_renderer_start_stenciling(true);
 		render_view(soutput, damage, con, colors, deco_data);
 		if (has_titlebar) {
 			render_titlebar(soutput, damage, con, floor(state->x), floor(state->y),
@@ -1288,6 +1340,21 @@ static void render_floating_container(struct sway_output *soutput,
 		} else if (state->border == B_PIXEL) {
 			render_top_border(soutput, damage, state, colors, deco_data.alpha, deco_data.corner_radius);
 		}
+
+		fx_renderer_close_stenciling(true);
+
+		// render shadow
+		if (con->shadow_enabled
+			&& config->shadow_blur_sigma > 0
+			&& config->shadow_color[3] > 0.0) {
+			struct wlr_box box = { state->x, state->y, state->width, state->height };
+			scale_box(&box, soutput->wlr_output->scale);
+			render_box_shadow(soutput, damage, &box, config->shadow_color,
+					config->shadow_blur_sigma, deco_data.corner_radius,
+					con->current.border_thickness);
+		}
+
+		fx_renderer_end_stenciling();
 	} else {
 		render_container(soutput, damage, con, state->focused);
 	}

sway/desktop/shaders/corner.frag

@@ -24,6 +24,11 @@ void main() {
     float smoothedAlphaInner = 1.0 - smoothstep(-1.0, 0.5, distance + half_thickness);
     gl_FragColor = mix(vec4(0), v_color, smoothedAlphaOuter - smoothedAlphaInner);
 
+    if ((v_color.a == 1.0 && gl_FragColor.a <= 0.5) || gl_FragColor.a <= 0.01) {
+        discard;
+        return;
+    }
+
     if (is_top_left && (center.y > 0.0 || center.x > 0.0)) {
         discard;
     } else if (is_top_right && (center.y > 0.0 || center.x < 0.0)) {

sway/desktop/shaders/quad.frag

@@ -4,4 +4,8 @@ varying vec2 v_texcoord;
 
 void main() {
     gl_FragColor = v_color;
+
+    if (gl_FragColor.a <= 0.01){
+        discard;
+    }
 }

sway/desktop/shaders/quad_round.frag

@@ -12,7 +12,8 @@ void main() {
     float distance = min(max(q.x,q.y),0.0) + length(max(q,0.0)) - radius;
     float smoothedAlpha = 1.0 - smoothstep(-1.0, 0.5, distance);
     gl_FragColor = mix(vec4(0), v_color, smoothedAlpha);
-    if (gl_FragColor.a == 0.0) {
+
+    if (gl_FragColor.a <= 0.01){
         discard;
     }
 }

sway/desktop/shaders/quad_round_tl.frag

@@ -11,4 +11,8 @@ void main() {
     float distance = min(max(q.x,q.y),0.0) + length(max(q,0.0)) - radius;
     float smoothedAlpha = 1.0 - smoothstep(-1.0, 0.5, distance);
     gl_FragColor = mix(vec4(0), v_color, smoothedAlpha);
+
+    if (gl_FragColor.a <= 0.01){
+        discard;
+    }
 }

sway/desktop/shaders/quad_round_tr.frag

@@ -11,4 +11,8 @@ void main() {
     float distance = min(max(q.x,q.y),0.0) + length(max(q,0.0)) - radius;
     float smoothedAlpha = 1.0 - smoothstep(-1.0, 0.5, distance);
     gl_FragColor = mix(vec4(0), v_color, smoothedAlpha);
+
+    if (gl_FragColor.a <= 0.01){
+        discard;
+    }
 }

sway/desktop/shaders/tex_external.frag

@@ -34,4 +34,8 @@ void main() {
             gl_FragColor = mix(vec4(0), gl_FragColor, smooth);
         }
     }
+
+    if (gl_FragColor.a <= 0.01){
+        discard;
+    }
 }

sway/desktop/shaders/tex_rgba.frag

@@ -32,4 +32,8 @@ void main() {
             gl_FragColor = mix(vec4(0), gl_FragColor, smooth);
         }
     }
+
+    if (gl_FragColor.a <= 0.01){
+        discard;
+    }
 }

sway/desktop/shaders/tex_rgbx.frag

@@ -31,4 +31,8 @@ void main() {
             gl_FragColor = mix(vec4(0), gl_FragColor, smooth);
         }
     }
+
+    if (gl_FragColor.a <= 0.01){
+        discard;
+    }
 }