Add color correction to the gles shader

data/debayer.frag

@@ -5,6 +5,7 @@ precision highp float;
 uniform sampler2D texture;
 uniform mat3 color_matrix;
 uniform float inv_gamma;
+uniform float blacklevel;
 #ifdef BITS_10
 uniform float row_length;
 uniform float padding_ratio;
@@ -59,16 +60,9 @@ main()
         vec3 color = vec3(samples.x, (samples.y + samples.z) / 2.0, samples.w);
 #endif
 
-        // Some crude blacklevel correction to make the preview a bit nicer, this
-        // should be an uniform
-        vec3 corrected = color - 0.02;
-
-        /*
-        // Fast SRGB estimate. See https://mimosa-pudica.net/fast-gamma/
-        vec3 gamma_color =
-                (vec3(1.138) * inversesqrt(corrected) - vec3(0.138)) * corrected;
-        */
-        color = color * color_matrix;
+        color -= blacklevel;
+        color *= 1.0-(1.0/blacklevel);
+        color *= color_matrix;
 
         vec3 gamma_color = pow(color, vec3(inv_gamma));
         gl_FragColor = vec4(gamma_color, 1);

src/gles2_debayer.c

@@ -9,22 +9,6 @@
 #define VERTEX_ATTRIBUTE 0
 #define TEX_COORD_ATTRIBUTE 1
 
-struct _GLES2Debayer {
-        int format;
-
-        GLuint frame_buffer;
-        GLuint program;
-        GLuint uniform_transform;
-        GLuint uniform_pixel_size;
-        GLuint uniform_padding_ratio;
-        GLuint uniform_texture;
-        GLuint uniform_color_matrix;
-        GLuint uniform_row_length;
-        GLuint uniform_inv_gamma;
-
-        GLuint quad;
-};
-
 GLES2Debayer *
 gles2_debayer_new(int format)
 {
@@ -79,6 +63,7 @@ gles2_debayer_new(int format)
         self->uniform_color_matrix =
                 glGetUniformLocation(self->program, "color_matrix");
         self->uniform_inv_gamma = glGetUniformLocation(self->program, "inv_gamma");
+        self->uniform_blacklevel = glGetUniformLocation(self->program, "blacklevel");
         if (libmegapixels_format_bits_per_pixel(self->format) == 10)
                 self->uniform_row_length =
                         glGetUniformLocation(self->program, "row_length");
@@ -109,6 +94,39 @@ gles2_debayer_use(GLES2Debayer *self)
         gl_util_bind_quad(self->quad);
 }
 
+void
+gles2_debayer_set_shading(GLES2Debayer *self,
+                          float red,
+                          float blue,
+                          float blacklevel)
+{
+        if (self->forward_matrix[0]) {
+                float wb[9] = {
+                        // clang-format off
+                        red, 0,   0,
+                        0,   1,   0,
+                        0,   0, blue,
+                        // clang-format on
+                };
+                float colormat[9];
+                float xyz[9];
+                float xyzd65[9];
+                multiply_matrices(wb, self->forward_matrix, xyz);
+                multiply_matrices(xyz, XYZD50_to_D65, xyzd65);
+                multiply_matrices(xyzd65, XYZD65_to_sRGB, colormat);
+                glUniformMatrix3fv(
+                        self->uniform_color_matrix, 1, GL_FALSE, colormat);
+
+        } else {
+                glUniformMatrix3fv(
+                        self->uniform_color_matrix, 1, GL_FALSE, IDENTITY);
+        }
+        check_gl();
+
+        glUniform1f(self->uniform_blacklevel, blacklevel);
+        check_gl();
+}
+
 void
 gles2_debayer_configure(GLES2Debayer *self,
                         const uint32_t dst_width,
@@ -153,26 +171,13 @@ gles2_debayer_configure(GLES2Debayer *self,
                 }
         }
         glUniform1f(self->uniform_inv_gamma, 1.0f / gamma);
+        check_gl();
 
-        if (calibration.forward_matrix_1[0]) {
-                float colormat[9];
-                float xyz[9];
-                multiply_matrices(calibration.forward_matrix_1, XYZD50_to_D65, xyz);
-                multiply_matrices(xyz, XYZD65_to_sRGB, colormat);
-                glUniformMatrix3fv(
-                        self->uniform_color_matrix, 1, GL_FALSE, colormat);
-        } else {
-                static const GLfloat identity[9] = {
-                        // clang-format off
-			1, 0, 0,
-			0, 1, 0,
-			0, 0, 1,
-                        // clang-format on
-                };
-                glUniformMatrix3fv(
-                        self->uniform_color_matrix, 1, GL_FALSE, identity);
+        for (int i = 0; i < 9; i++) {
+                self->forward_matrix[i] = calibration.forward_matrix_1[i];
         }
-        check_gl();
+
+        gles2_debayer_set_shading(self, 1.0f, 1.0f, 0.0f);
 
         GLuint row_length =
                 libmegapixels_mode_width_to_bytes(self->format, src_width);

src/gles2_debayer.h

@@ -4,7 +4,23 @@
 #include <assert.h>
 #include <stdio.h>
 
-typedef struct _GLES2Debayer GLES2Debayer;
+typedef struct {
+        int format;
+        float forward_matrix[9];
+
+        GLuint frame_buffer;
+        GLuint program;
+        GLint uniform_transform;
+        GLint uniform_pixel_size;
+        GLint uniform_padding_ratio;
+        GLint uniform_texture;
+        GLint uniform_color_matrix;
+        GLint uniform_row_length;
+        GLint uniform_inv_gamma;
+        GLint uniform_blacklevel;
+
+        GLuint quad;
+} GLES2Debayer;
 
 GLES2Debayer *gles2_debayer_new(int format);
 
@@ -13,12 +29,17 @@ void gles2_debayer_free(GLES2Debayer *self);
 void gles2_debayer_use(GLES2Debayer *self);
 
 void gles2_debayer_configure(GLES2Debayer *self,
-                             const uint32_t dst_width,
-                             const uint32_t dst_height,
-                             const uint32_t src_width,
-                             const uint32_t src_height,
-                             const uint32_t rotation,
-                             const bool mirrored,
+                             uint32_t dst_width,
+                             uint32_t dst_height,
+                             uint32_t src_width,
+                             uint32_t src_height,
+                             uint32_t rotation,
+                             bool mirrored,
                              struct MPCameraCalibration calibration);
 
+void gles2_debayer_set_shading(GLES2Debayer *self,
+                               float red,
+                               float blue,
+                               float blacklevel);
+
 void gles2_debayer_process(GLES2Debayer *self, GLuint dst_id, GLuint source_id);

src/matrix.c

@@ -1,15 +1,16 @@
+#include "matrix.h"
 #include <stdio.h>
 
 void
 print_matrix(float m[9])
 {
-        printf(" [%.2f  %.2f  %.2f] \n", m[0], m[1], m[2]);
-        printf(" [%.2f  %.2f  %.2f] \n", m[3], m[4], m[5]);
-        printf(" [%.2f  %.2f  %.2f] \n\n", m[6], m[7], m[8]);
+        printf(" [%+.2f  %+.2f  %+.2f] \n", m[0], m[1], m[2]);
+        printf(" [%+.2f  %+.2f  %+.2f] \n", m[3], m[4], m[5]);
+        printf(" [%+.2f  %+.2f  %+.2f] \n\n", m[6], m[7], m[8]);
 }
 
 void
-multiply_matrices(float a[9], float b[9], float out[9])
+multiply_matrices(const float a[9], const float b[9], float out[9])
 {
         // zero out target matrix
         for (int i = 0; i < 9; i++) {
@@ -24,3 +25,28 @@ multiply_matrices(float a[9], float b[9], float out[9])
                 }
         }
 }
+
+void
+invert_matrix(const float in[9], float out[9])
+{
+        float det = in[0] * (in[4] * in[8] - in[5] * in[7]) -
+                    in[1] * (in[3] * in[8] - in[5] * in[6]) +
+                    in[2] * (in[3] * in[7] - in[4] * in[7]);
+        out[0] = (in[4] * in[8] - in[7] * in[5]) / det;
+        out[1] = (in[7] * in[2] - in[1] * in[8]) / det;
+        out[2] = (in[1] * in[5] - in[4] * in[2]) / det;
+        out[3] = (in[6] * in[5] - in[3] * in[8]) / det;
+        out[4] = (in[0] * in[8] - in[6] * in[5]) / det;
+        out[5] = (in[3] * in[2] - in[0] * in[5]) / det;
+        out[6] = (in[3] * in[7] - in[6] * in[4]) / det;
+        out[7] = (in[6] * in[1] - in[0] * in[7]) / det;
+        out[8] = (in[0] * in[4] - in[3] * in[1]) / det;
+}
+
+void
+transpose_matrix(const float in[9], float out[9])
+{
+        for (int i = 0; i < 3; ++i)
+                for (int j = 0; j < 3; ++j)
+                        out[i + j * 3] = in[j + i * 3];
+}

src/matrix.h

@@ -1,34 +1,33 @@
-static float XYZD50_to_D65[] = { 0.9555766f,  -0.0230393f, 0.0631636f,
-                                -0.0282895f, 1.0099416f,  0.0210077f,
-                                 0.0122982f,  -0.0204830f, 1.3299098f };
+#pragma once
 
-static float XYZD65_to_sRGB[] = {  3.2406f,  -1.5372f,   -0.4986f,
-                                  -0.9689f,   1.8758f,    0.0415f,
-                                   0.0557f,  -0.2040f,    1.0570f };
+static float IDENTITY[9] = {
+        // clang-format off
+        1, 0, 0,
+        0, 1, 0,
+        0, 0, 1,
+        // clang-format on
+};
 
-void multiply_matrices(float a[9], float b[9], float out[9]);
+static float XYZD50_to_D65[] = {
+        // clang-format off
+         0.9555766f,  -0.0230393f, 0.0631636f,
+        -0.0282895f,   1.0099416f, 0.0210077f,
+         0.0122982f,  -0.0204830f, 1.3299098f
+        // clang-format on
+};
 
-void
-invert_matrix(const float in[9], float out[9])
-{
-        float det = in[0] * (in[4] * in[8] - in[5] * in[7]) -
-                    in[1] * (in[3] * in[8] - in[5] * in[6]) +
-                    in[2] * (in[3] * in[7] - in[4] * in[7]);
-        out[0] = (in[4] * in[8] - in[7] * in[5]) / det;
-        out[1] = (in[7] * in[2] - in[1] * in[8]) / det;
-        out[2] = (in[1] * in[5] - in[4] * in[2]) / det;
-        out[3] = (in[6] * in[5] - in[3] * in[8]) / det;
-        out[4] = (in[0] * in[8] - in[6] * in[5]) / det;
-        out[5] = (in[3] * in[2] - in[0] * in[5]) / det;
-        out[6] = (in[3] * in[7] - in[6] * in[4]) / det;
-        out[7] = (in[6] * in[1] - in[0] * in[7]) / det;
-        out[8] = (in[0] * in[4] - in[3] * in[1]) / det;
-}
+static float XYZD65_to_sRGB[] = {
+        // clang-format off
+         3.2406f,  -1.5372f,   -0.4986f,
+        -0.9689f,   1.8758f,    0.0415f,
+         0.0557f,  -0.2040f,    1.0570f
+        // clang-format on
+};
 
-void
-transpose_matrix(const float in[9], float out[9])
-{
-        for (int i = 0; i < 3; ++i)
-                for (int j = 0; j < 3; ++j)
-                        out[i + j * 3] = in[j + i * 3];
-}
+void print_matrix(float m[9]);
+
+void multiply_matrices(const float a[9], const float b[9], float out[9]);
+
+void invert_matrix(const float in[9], float out[9]);
+
+void transpose_matrix(const float in[9], float out[9]);

src/process_pipeline.c

@@ -473,6 +473,11 @@ process_image_for_preview(const uint8_t *image)
                              center);
                 libmegapixels_aaa_software_statistics(
                         center, width, height, &state_proc.stats);
+
+                state_proc.red += (state_proc.stats.whitebalance * -0.02f) + (state_proc.stats.tint * 0.01f);
+                state_proc.blue += (state_proc.stats.whitebalance * +0.02f) + (state_proc.stats.tint * 0.01f);
+                state_proc.blacklevel += state_proc.stats.blacklevel * 0.01f;
+                gles2_debayer_set_shading(gles2_debayer, state_proc.red, state_proc.blue, state_proc.blacklevel);
         }
 
         // Create a thumbnail from the preview for the last capture
@@ -933,6 +938,9 @@ on_output_changed(bool format_changed)
                 gles2_debayer_use(gles2_debayer);
         }
 
+        state_proc.blacklevel = 0.0f;
+        state_proc.red = 1.0f;
+        state_proc.blue = 1.0f;
         gles2_debayer_configure(gles2_debayer,
                                 output_buffer_width,
                                 output_buffer_height,

src/state.h

@@ -94,4 +94,8 @@ typedef struct state_proc {
         controlstate focus;
 
         struct MPCameraCalibration calibration;
+
+        float red;
+        float blue;
+        float blacklevel;
 } mp_state_proc;