various security fixes and precautionary checks

Originally committed as revision 3822 to svn://svn.ffmpeg.org/ffmpeg/trunk

libavcodec/ratecontrol.c

@@ -74,6 +74,8 @@ int ff_rate_control_init(MpegEncContext *s)
             p= strchr(p+1, ';');
         }
         i+= s->max_b_frames;
+        if(i<=0 || i>=INT_MAX / sizeof(RateControlEntry))
+            return -1;
         rcc->entry = (RateControlEntry*)av_mallocz(i*sizeof(RateControlEntry));
         rcc->num_entries= i;