Fix avpicture_get_size for non-paletted formats with a helper palette

to not include the size of that palette.
Also clarify its documentation.

Originally committed as revision 18106 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/avcodec.h

@@ -2676,6 +2676,7 @@ void avpicture_free(AVPicture *picture);
  * If a planar format is specified, several pointers will be set pointing to
  * the different picture planes and the line sizes of the different planes
  * will be stored in the lines_sizes array.
+ * Call with ptr == NULL to get the required size for the ptr buffer.
  *
  * @param picture AVPicture whose fields are to be filled in
  * @param ptr Buffer which will contain or contains the actual image data
@@ -2692,11 +2693,14 @@ int avpicture_layout(const AVPicture* src, int pix_fmt, int width, int height,
 /**
  * Calculate the size in bytes that a picture of the given width and height
  * would occupy if stored in the given picture format.
+ * Note that this returns the size of a compact representation as generated
+ * by avpicture_layout, which can be smaller than the size required for e.g.
+ * avpicture_fill.
  *
  * @param pix_fmt the given picture format
  * @param width the width of the image
  * @param height the height of the image
- * @return Image data size in bytes
+ * @return Image data size in bytes or -1 on error (e.g. too large dimensions).
  */
 int avpicture_get_size(int pix_fmt, int width, int height);
 void avcodec_get_chroma_sub_sample(int pix_fmt, int *h_shift, int *v_shift);

libavcodec/imgconvert.c

@@ -786,6 +786,17 @@ int avpicture_layout(const AVPicture* src, int pix_fmt, int width, int height,
 int avpicture_get_size(int pix_fmt, int width, int height)
 {
     AVPicture dummy_pict;
+    if(avcodec_check_dimensions(NULL, width, height))
+        return -1;
+    switch (pix_fmt) {
+    case PIX_FMT_RGB8:
+    case PIX_FMT_BGR8:
+    case PIX_FMT_RGB4_BYTE:
+    case PIX_FMT_BGR4_BYTE:
+    case PIX_FMT_GRAY8:
+        // do not include palette for these pseudo-paletted formats
+        return width * height;
+    }
     return avpicture_fill(&dummy_pict, NULL, pix_fmt, width, height);
 }
 
@@ -1125,7 +1136,7 @@ int avpicture_alloc(AVPicture *picture,
     int size;
     void *ptr;
 
-    size = avpicture_get_size(pix_fmt, width, height);
+    size = avpicture_fill(picture, NULL, pix_fmt, width, height);
     if(size<0)
         goto fail;
     ptr = av_malloc(size);

tests/libav.regression.ref

@@ -47,7 +47,7 @@ af78858062599fcbba049e4a02588a15 *./tests/data/b-libav.mov
 ./tests/data/b-pbmpipe.pbm CRC=0x806e17d8
 aff140ce80a1c86c1bf54118ad23da7b *./tests/data/b-pgmpipe.pgm
 2534775 ./tests/data/b-pgmpipe.pgm
-./tests/data/b-pgmpipe.pgm CRC=0x0e82c482
+./tests/data/b-pgmpipe.pgm CRC=0xf485870f
 9169b1f1ca56f01a6e1f5041572aa1d4 *./tests/data/b-ppmpipe.ppm
 7603575 ./tests/data/b-ppmpipe.ppm
 ./tests/data/b-ppmpipe.ppm CRC=0x79bd6ce6
@@ -56,7 +56,7 @@ aff140ce80a1c86c1bf54118ad23da7b *./tests/data/b-pgmpipe.pgm
 b977a4fedff90a79baf70c8e02986820 *./tests/data/b-libav.y4m
 3801810 ./tests/data/b-libav.y4m
 0a6d74b54396884f117669965b57d3b5 *./tests/data/b-libav02.pgm
-./tests/data/b-libav%02d.pgm CRC=0xc8032eb1
+./tests/data/b-libav%02d.pgm CRC=0x7e552eb1
 101391 ./tests/data/b-libav02.pgm
 dbe42bd8d9ca0acbd2673bd739705f0f *./tests/data/b-libav02.ppm
 ./tests/data/b-libav%02d.ppm CRC=0x6f775c0d