sws_scale gray->float: what am i doing wrong?

190 views Asked by At

I am trying to convert a frame in gray8 format to grayf32 and gbrpf32 formats. The code below works for grayf32 but not for gbrpf32. What am I doing wrong in the below code?

AVFrame* gray = av_frame_alloc();
gray->format = AV_PIX_FMT_GRAY8;
gray->width = 4;
gray->height = 1;
gray->color_range = AVCOL_RANGE_JPEG;
av_frame_get_buffer(gray, 0);

uint8_t grayVals[4] = { 0, 85, 170, 255 };
memcpy(gray->data[0], grayVals, 4);

AVFrame* floatGray = av_frame_alloc();
floatGray->format = AV_PIX_FMT_GRAYF32;
floatGray->width = gray->width;
floatGray->height = gray->height;
floatGray->color_range = AVCOL_RANGE_JPEG;
av_frame_get_buffer(floatGray, 0);

SwsContext* sws_ctx = sws_getContext(
    gray->width, gray->height, AV_PIX_FMT_GRAY8,
    floatGray->width, floatGray->height, AV_PIX_FMT_GRAYF32,
    SWS_BILINEAR,
    NULL, NULL, NULL);
av_opt_set_int(sws_ctx, "src_range", 1, 0);
av_opt_set_int(sws_ctx, "dst_range", 1, 0);

int ret = sws_scale(sws_ctx, (const uint8_t* const*)gray->data,
    gray->linesize, 0, gray->height, floatGray->data,
    floatGray->linesize);
if (ret < 0)
    printf("error (1): %d\n", ret);

sws_freeContext(sws_ctx);


AVFrame* floatRGB = av_frame_alloc();
floatRGB->format = AV_PIX_FMT_GBRPF32;
floatRGB->width = gray->width;
floatRGB->height = gray->height;
floatRGB->color_range = AVCOL_RANGE_JPEG;
av_frame_get_buffer(floatRGB, 0);

sws_ctx = sws_getContext(
    gray->width, gray->height, AV_PIX_FMT_GRAY8,
    floatRGB->width, floatRGB->height, AV_PIX_FMT_GBRPF32,
    SWS_BILINEAR,
    NULL, NULL, NULL);
av_opt_set_int(sws_ctx, "src_range", 1, 0);
av_opt_set_int(sws_ctx, "dst_range", 1, 0);

ret = sws_scale(sws_ctx, (const uint8_t* const*)gray->data,
    gray->linesize, 0, gray->height, floatRGB->data,
    floatRGB->linesize);
if (ret < 0)
    printf("error (1): %d\n", ret);
sws_freeContext(sws_ctx);

uint8_t* input = gray->data[0];
float* out = (float*)floatGray->data[0];
float* outG = (float*)floatRGB->data[0];
float* outB = (float*)floatRGB->data[1];
float* outR = (float*)floatRGB->data[2];

printf("input: \n");
for (int i = 0; i < 4; ++i)
    printf("%d, ", input[i]);
printf("\ngray floats: \n");
for (int i = 0; i < 4; ++i)
    printf("%.5f, ", out[i]);
printf("\nRGB floats (G): \n");
for (int i = 0; i < 4; ++i)
    printf("%.5f, ", outG[i]);
printf("\nRGB floats (B): \n");
for (int i = 0; i < 4; ++i)
    printf("%.5f, ", outB[i]);
printf("\nRGB floats (R): \n");
for (int i = 0; i < 4; ++i)
    printf("%.5f, ", outR[i]);

This is the output I get:

input:
0, 85, 170, 255,
gray floats:
0.00000, 0.33333, 0.66667, 1.00000,
RGB floats (G):
0.52912, 0.86114, 1.00000, 1.00000,
RGB floats (B):
0.00000, 0.00000, 0.00000, 0.11012,
RGB floats (R):
0.00000, 0.00000, 0.00000, 0.29512,

While I would have expected 0.00000, 0.33333, 0.66667, 1.00000 for all the channels of the RGB float, it's clearly not what I get. FWIW, if I use an rgb24 instead of gray8 as input (so rgb24->gray32f and rgb24->gbrpf32), then it does work mostly correctly for both, except that for the grayf32 format the wrong color range seems to be used (despite being set) and there is some round-off error for gbrpf32 (code not shown):

gray floats:
0.06250, 0.34766, 0.63282, 0.91798,
RGB floats (G):
0.00000, 0.33205, 0.66410, 0.99615,
RGB floats (B):
0.00000, 0.33205, 0.66410, 0.99615,
RGB floats (R):
0.00000, 0.33205, 0.66410, 0.99615,
0

There are 0 answers