commit 25b27a9c167302769db512a9e32c66323bc7904c
author Mark Harris <mark.hsj@gmail.com> Thu Nov 27 08:48:09 2014 -0800
committer Jean-Marc Valin <jmvalin@jmvalin.ca> Thu Nov 27 12:04:14 2014 -0500
tree 2c0df6c184b180fcf252f94a6389a23e42d04666
parent d10755e95c49f5ec925e70598877c82ad865a189

multistream: improve arg check

Avoid undefined behavior (signed arithmetic overflow) or
implementation-defined behavior (malloc(0)) on out-of-range arguments,
e.g. opus_multistream_encoder_create(48000, 2, 2147483647, 1, ...)
or opus_multistream_surround_encoder_create(48000, 3, 0, ...).

Signed-off-by: Jean-Marc Valin <jmvalin@jmvalin.ca>

src/opus_multistream_encoder.c

diff --git a/src/opus_multistream_encoder.c b/src/opus_multistream_encoder.c
index 3787194..6e87337 100644
--- a/src/opus_multistream_encoder.c
+++ b/src/opus_multistream_encoder.c
@@ -408,7 +408,7 @@
    char *ptr;
 
    if ((channels>255) || (channels<1) || (coupled_streams>streams) ||
-       (coupled_streams+streams>255) || (streams<1) || (coupled_streams<0))
+       (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams))
       return OPUS_BAD_ARG;
 
    st->layout.nb_channels = channels;
@@ -530,7 +530,7 @@
    int ret;
    OpusMSEncoder *st;
    if ((channels>255) || (channels<1) || (coupled_streams>streams) ||
-       (coupled_streams+streams>255) || (streams<1) || (coupled_streams<0))
+       (streams<1) || (coupled_streams<0) || (streams>255-coupled_streams))
    {
       if (error)
          *error = OPUS_BAD_ARG;
@@ -566,6 +566,7 @@
 )
 {
    int ret;
+   opus_int32 size;
    OpusMSEncoder *st;
    if ((channels>255) || (channels<1))
    {
@@ -573,7 +574,14 @@
          *error = OPUS_BAD_ARG;
       return NULL;
    }
-   st = (OpusMSEncoder *)opus_alloc(opus_multistream_surround_encoder_get_size(channels, mapping_family));
+   size = opus_multistream_surround_encoder_get_size(channels, mapping_family);
+   if (!size)
+   {
+      if (error)
+         *error = OPUS_UNIMPLEMENTED;
+      return NULL;
+   }
+   st = (OpusMSEncoder *)opus_alloc(size);
    if (st==NULL)
    {
       if (error)