Subpixel‐aware downscaling filters

src/Backends/DRMBackend.cpp

@@ -3451,7 +3451,7 @@ namespace gamescope
 
 			bool bLayer0ScreenSize = close_enough(pFrameInfo->layers[0].scale.x, 1.0f) && close_enough(pFrameInfo->layers[0].scale.y, 1.0f);
 
-			bool bNeedsCompositeFromFilter = (g_upscaleFilter == GamescopeUpscaleFilter::NEAREST || g_upscaleFilter == GamescopeUpscaleFilter::PIXEL) && !bLayer0ScreenSize;
+			bool bNeedsCompositeFromFilter = (g_upscaleFilter == GamescopeUpscaleFilter::NEAREST || g_upscaleFilter == GamescopeUpscaleFilter::PIXEL || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_RGB || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_OLED || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_VBGR || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_QDOLED) && !bLayer0ScreenSize;
 
 			bool bNeedsFullComposite = false;
 			bNeedsFullComposite |= cv_composite_force;
@@ -4009,4 +4009,3 @@ int HackyDRMPresent( const FrameInfo_t *pFrameInfo, bool bAsync )
 {
 	return static_cast<gamescope::CDRMBackend *>( GetBackend() )->Present( pFrameInfo, bAsync );
 }
-

src/Backends/OpenVRBackend.cpp

@@ -1494,7 +1494,7 @@ namespace gamescope
         // TODO: Dedupe some of this composite check code between us and drm.cpp
         bool bLayer0ScreenSize = close_enough(pFrameInfo->layers[0].scale.x, 1.0f) && close_enough(pFrameInfo->layers[0].scale.y, 1.0f);
 
-        bool bNeedsCompositeFromFilter = (g_upscaleFilter == GamescopeUpscaleFilter::NEAREST || g_upscaleFilter == GamescopeUpscaleFilter::PIXEL) && !bLayer0ScreenSize;
+        bool bNeedsCompositeFromFilter = (g_upscaleFilter == GamescopeUpscaleFilter::NEAREST || g_upscaleFilter == GamescopeUpscaleFilter::PIXEL || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_RGB || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_OLED || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_VBGR || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_QDOLED) && !bLayer0ScreenSize;
 
         bNeedsFullComposite |= cv_composite_force;
         bNeedsFullComposite |= pFrameInfo->useFSRLayer0;

src/Backends/WaylandBackend.cpp

@@ -1057,7 +1057,7 @@ namespace gamescope
             // TODO: Dedupe some of this composite check code between us and drm.cpp
             bool bLayer0ScreenSize = close_enough(pFrameInfo->layers[0].scale.x, 1.0f) && close_enough(pFrameInfo->layers[0].scale.y, 1.0f);
 
-            bool bNeedsCompositeFromFilter = (g_upscaleFilter == GamescopeUpscaleFilter::NEAREST || g_upscaleFilter == GamescopeUpscaleFilter::PIXEL) && !bLayer0ScreenSize;
+            bool bNeedsCompositeFromFilter = (g_upscaleFilter == GamescopeUpscaleFilter::NEAREST || g_upscaleFilter == GamescopeUpscaleFilter::PIXEL || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_RGB || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_OLED || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_VBGR || g_upscaleFilter == GamescopeUpscaleFilter::SUBPIXEL_QDOLED) && !bLayer0ScreenSize;
 
             bNeedsFullComposite |= cv_composite_force;
             bNeedsFullComposite |= pFrameInfo->useFSRLayer0;

src/main.cpp

@@ -172,9 +172,13 @@ const char usage[] =
 	"  -r, --nested-refresh           game refresh rate (frames per second)\n"
 	"  -m, --max-scale                maximum scale factor\n"
 	"  -S, --scaler                   upscaler type (auto, integer, fit, fill, stretch)\n"
-	"  -F, --filter                   upscaler filter (linear, nearest, fsr, nis, pixel)\n"
+"  -F, --filter                   upscaler/downscaler filter (linear, nearest, fsr, nis, pixel, subpixel_rgb, subpixel_vbgr, subpixel_oled, subpixel_qdoled)\n"
 	"                                     fsr => AMD FidelityFX™ Super Resolution 1.0\n"
 	"                                     nis => NVIDIA Image Scaling v1.0.3\n"
+	"                                     subpixel_rgb => horizontal RGB subpixel layout (3:1 downscale)\n"
+	"                                     subpixel_vbgr => vertical BGR subpixel layout (3:1 downscale)\n"
+	"                                     subpixel_oled => RG/B subpixel layout (2:1 downscale)\n"
+	"                                     subpixel_qdoled => G/RB subpixel layout (2:1 downscale)\n"
 	"  --sharpness, --fsr-sharpness   upscaler sharpness from 0 (max) to 20 (min)\n"
 	"  --expose-wayland               support wayland clients using xdg-shell\n"
 	"  -s, --mouse-sensitivity        multiply mouse movement by given decimal number\n"
@@ -410,6 +414,14 @@ static enum GamescopeUpscaleFilter parse_upscaler_filter(const char *str)
 		return GamescopeUpscaleFilter::NIS;
 	} else if (strcmp(str, "pixel") == 0) {
 		return GamescopeUpscaleFilter::PIXEL;
+	} else if (strcmp(str, "subpixel_rgb") == 0) {
+		return GamescopeUpscaleFilter::SUBPIXEL_RGB;
+	} else if (strcmp(str, "subpixel_oled") == 0) {
+		return GamescopeUpscaleFilter::SUBPIXEL_OLED;
+	} else if (strcmp(str, "subpixel_vbgr") == 0) {
+		return GamescopeUpscaleFilter::SUBPIXEL_VBGR;
+	} else if (strcmp(str, "subpixel_qdoled") == 0) {
+		return GamescopeUpscaleFilter::SUBPIXEL_QDOLED;
 	} else {
 		fprintf( stderr, "gamescope: invalid value for --filter\n" );
 		exit(1);

src/main.hpp

@@ -36,6 +36,10 @@ enum class GamescopeUpscaleFilter : uint32_t
     FSR,
     NIS,
     PIXEL,
+    SUBPIXEL_RGB,
+    SUBPIXEL_OLED,
+    SUBPIXEL_VBGR,
+    SUBPIXEL_QDOLED,
 
     FROM_VIEW = 0xF, // internal
 };
@@ -72,4 +76,3 @@ extern int g_nXWaylandCount;
 
 extern uint32_t g_preferVendorID;
 extern uint32_t g_preferDeviceID;
-

src/rendervulkan.cpp

@@ -3635,6 +3635,96 @@ float g_flInternalDisplayBrightnessNits = 500.0f;
 float g_flHDRItmSdrNits = 100.f;
 float g_flHDRItmTargetNits = 1000.f;
 
+struct SubpixelFilterDefinition
+{
+	GamescopeUpscaleFilter eFilter;
+	vec2_t downscaleRatio;
+	const char *pName;
+};
+
+static constexpr std::array<SubpixelFilterDefinition, 4> g_SubpixelFilterDefinitions = {{
+	{ GamescopeUpscaleFilter::SUBPIXEL_RGB,   { 3.0f, 3.0f },   "horizontal RGB" },
+	{ GamescopeUpscaleFilter::SUBPIXEL_OLED,  { 2.0f, 2.0f },   "RG/B OLED" },
+	{ GamescopeUpscaleFilter::SUBPIXEL_VBGR,  { 3.0f, 3.0f },   "vertical BGR" },
+	{ GamescopeUpscaleFilter::SUBPIXEL_QDOLED,{ 2.0f, 2.0f },   "G/RB QD-OLED" },
+}};
+
+static const SubpixelFilterDefinition *FindSubpixelFilterDefinition( GamescopeUpscaleFilter eFilter )
+{
+	for ( const auto &definition : g_SubpixelFilterDefinitions )
+	{
+		if ( definition.eFilter == eFilter )
+			return &definition;
+	}
+
+	return nullptr;
+}
+
+static GamescopeUpscaleFilter GetLayerShaderFilter( const FrameInfo_t::Layer_t &layer )
+{
+	if ( layer.isScreenSize() || ( layer.filter == GamescopeUpscaleFilter::LINEAR && layer.viewConvertsToLinearAutomatically() ) )
+		return GamescopeUpscaleFilter::FROM_VIEW;
+
+	if ( const auto *definition = FindSubpixelFilterDefinition( layer.filter ) )
+	{
+		static int s_lastState[ g_SubpixelFilterDefinitions.size() ] = { -1, -1, -1, -1 };
+		size_t idx = definition - g_SubpixelFilterDefinitions.data();
+
+		float dimRatioX = 0.0f;
+		float dimRatioY = 0.0f;
+		if ( layer.tex )
+		{
+			dimRatioX = layer.tex->width()  / (float)std::max(1u, currentOutputWidth);
+			dimRatioY = layer.tex->height() / (float)std::max(1u, currentOutputHeight);
+		}
+
+		auto scaleToRatio = []( float s ) -> float {
+			if ( s == 0.0f )
+				return 0.0f;
+			return s >= 1.0f ? s : (1.0f / s);
+		};
+
+		float scaleRatioX = scaleToRatio( layer.scale.x );
+		float scaleRatioY = scaleToRatio( layer.scale.y );
+
+		float observedX = dimRatioX > 0.0f ? dimRatioX : scaleRatioX;
+		float observedY = dimRatioY > 0.0f ? dimRatioY : scaleRatioY;
+
+		bool ratioOk =
+			close_enough( observedX, definition->downscaleRatio.x ) &&
+			close_enough( observedY, definition->downscaleRatio.y );
+
+		int state = ratioOk ? 1 : 0;
+		if ( state != s_lastState[idx] )
+		{
+			if ( ratioOk )
+			{
+				vk_log.infof( "Subpixel %s filter active: scale=(%.3f, %.3f) ratio=(%.3f, %.3f) tex=%ux%u out=%ux%u target=(%.1f, %.1f)",
+					definition->pName,
+					layer.scale.x, layer.scale.y, observedX, observedY,
+					layer.tex ? layer.tex->width() : 0, layer.tex ? layer.tex->height() : 0,
+					currentOutputWidth, currentOutputHeight,
+					definition->downscaleRatio.x, definition->downscaleRatio.y );
+			}
+			else
+			{
+				vk_log.warnf( "Subpixel %s filter disabled (ratio mismatch): scale=(%.3f, %.3f) ratio=(%.3f, %.3f) tex=%ux%u out=%ux%u target=(%.1f, %.1f)",
+					definition->pName,
+					layer.scale.x, layer.scale.y, observedX, observedY,
+					layer.tex ? layer.tex->width() : 0, layer.tex ? layer.tex->height() : 0,
+					currentOutputWidth, currentOutputHeight,
+					definition->downscaleRatio.x, definition->downscaleRatio.y );
+			}
+			s_lastState[idx] = state;
+		}
+
+		if ( !ratioOk )
+			return GamescopeUpscaleFilter::LINEAR;
+	}
+
+	return layer.filter;
+}
+
 #pragma pack(push, 1)
 struct BlitPushData_t
 {
@@ -3664,10 +3754,7 @@ struct BlitPushData_t
 			scale[i] = layer->scale;
 			offset[i] = layer->offsetPixelCenter();
 			opacity[i] = layer->opacity;
-            if (layer->isScreenSize() || (layer->filter == GamescopeUpscaleFilter::LINEAR && layer->viewConvertsToLinearAutomatically()))
-                u_shaderFilter |= ((uint32_t)GamescopeUpscaleFilter::FROM_VIEW) << (i * 4);
-            else
-                u_shaderFilter |= ((uint32_t)layer->filter) << (i * 4);
+			u_shaderFilter |= ((uint32_t)GetLayerShaderFilter(*layer)) << (i * 4);
 
 			u_alphaMode |= ((uint32_t)layer->eAlphaBlendingMode) << ( i * 4 );
 
@@ -3804,10 +3891,8 @@ struct RcasPushData_t
 		{
 			const FrameInfo_t::Layer_t *layer = &frameInfo->layers[i];
 
-            if (i == 0 || layer->isScreenSize() || (layer->filter == GamescopeUpscaleFilter::LINEAR && layer->viewConvertsToLinearAutomatically()))
-                u_shaderFilter |= ((uint32_t)GamescopeUpscaleFilter::FROM_VIEW) << (i * 4);
-            else
-                u_shaderFilter |= ((uint32_t)layer->filter) << (i * 4);
+			GamescopeUpscaleFilter shaderFilter = i == 0 ? GamescopeUpscaleFilter::FROM_VIEW : GetLayerShaderFilter(*layer);
+			u_shaderFilter |= ((uint32_t)shaderFilter) << (i * 4);
 
 			u_alphaMode |= ((uint32_t)layer->eAlphaBlendingMode) << ( i * 4 );
 

src/shaders/composite.h

@@ -2,6 +2,7 @@
 
 #include "shaderfilter.h"
 #include "alphamode.h"
+#include "subpixel_scaler.h"
 
 vec4 sampleRegular(sampler2D tex, vec2 coord, uint colorspace) {
     vec4 color = textureLod(tex, coord, 0);
@@ -148,7 +149,9 @@ vec4 sampleBilinear(sampler2D tex, vec2 coord, uint colorspace, bool unnormalize
 
 vec4 sampleLayerEx(sampler2D layerSampler, uint offsetLayerIdx, uint colorspaceLayerIdx, vec2 uv, bool unnormalized) {
     vec2 coord = ((uv + u_offset[offsetLayerIdx]) * u_scale[offsetLayerIdx]);
-    vec2 texSize = textureSize(layerSampler, 0);
+    vec2 unnormalizedCoord = coord;
+    ivec2 texSizeInt = textureSize(layerSampler, 0);
+    vec2 texSize = vec2(texSizeInt);
 
     if (coord.x < 0.0f       || coord.y < 0.0f ||
         coord.x >= texSize.x || coord.y >= texSize.y) {
@@ -165,17 +168,45 @@ vec4 sampleLayerEx(sampler2D layerSampler, uint offsetLayerIdx, uint colorspaceL
 
     uint colorspace = get_layer_colorspace(colorspaceLayerIdx);
     vec4 color;
-    if (get_layer_shaderfilter(offsetLayerIdx) == filter_pixel) {
+    uint shaderFilter = get_layer_shaderfilter(offsetLayerIdx);
+
+    if (!unnormalized) {
+        if (shaderFilter == filter_subpixel_oled ||
+            shaderFilter == filter_subpixel_qdoled ||
+            shaderFilter == filter_subpixel_rgb ||
+            shaderFilter == filter_subpixel_vbgr)
+        {
+            shaderFilter = filter_nearest;
+        }
+    }
+
+    switch (shaderFilter) {
+    case filter_subpixel_oled:
+        SAMPLE_SUBPIXEL_OLED_FILTER(layerSampler, unnormalizedCoord, texSizeInt, u_scale[offsetLayerIdx], colorspace, color);
+        break;
+    case filter_subpixel_qdoled:
+        SAMPLE_SUBPIXEL_QDOLED_FILTER(layerSampler, unnormalizedCoord, texSizeInt, u_scale[offsetLayerIdx], colorspace, color);
+        break;
+    case filter_subpixel_rgb:
+        SAMPLE_SUBPIXEL_RGB_FILTER(layerSampler, unnormalizedCoord, texSizeInt, u_scale[offsetLayerIdx], colorspace, color);
+        break;
+    case filter_subpixel_vbgr:
+        SAMPLE_SUBPIXEL_VBGR_FILTER(layerSampler, unnormalizedCoord, texSizeInt, u_scale[offsetLayerIdx], colorspace, color);
+        break;
+    case filter_pixel: {
         vec2 output_res = texSize / u_scale[offsetLayerIdx];
         vec2 extent = max((texSize / output_res), vec2(1.0 / 256.0));
         color = sampleBandLimited(layerSampler, coord, unnormalized ? vec2(1.0f) : texSize, unnormalized ? vec2(1.0f) : vec2(1.0f) / texSize, extent, colorspace, unnormalized);
+        break;
     }
-    else if (get_layer_shaderfilter(offsetLayerIdx) == filter_linear_emulated) {
+    case filter_linear_emulated:
         color = sampleBilinear(layerSampler, coord, colorspace, unnormalized);
-    }
-    else {
+        break;
+    default:
         color = sampleRegular(layerSampler, coord, colorspace);
+        break;
     }
+
     // JoshA: AMDGPU applies 3x4 CTM like this, where A is 1.0, but it only affects .rgb.
     color.rgb = vec4(color.rgb, 1.0f) * u_ctm[colorspaceLayerIdx];
     color.rgb = apply_layer_color_mgmt(color.rgb, offsetLayerIdx, colorspace);

src/shaders/descriptor_set.h

@@ -21,6 +21,10 @@ const int filter_nearest = 1;
 const int filter_fsr = 2;
 const int filter_nis = 3;
 const int filter_pixel = 4;
+const int filter_subpixel_rgb = 5;
+const int filter_subpixel_oled = 6;
+const int filter_subpixel_vbgr = 7;
+const int filter_subpixel_qdoled = 8;
 const int filter_from_view = 255;
 
 const int EOTF_Gamma22 = 0;