faad2: Backport some security fixes from upstream.

CVE-2018-20194:
https://github.com/knik0/faad2/commit/6b4a7cde30f2e2cb03e78ef476cc73179cfffda3.patch

CVE-2018-20362:
https://github.com/knik0/faad2/commit/466b01d504d7e45f1e9169ac90b3e34ab94aed14.patch

Misc buffer overflows:
https://github.com/knik0/faad2/commit/942c3e0aee748ea6fe97cb2c1aa5893225316174.patch


(nia)

$NetBSD: patch-CVE-2018-20194,v 1.1 2019/07/11 09:03:35 nia Exp $

user passed f_table_lim contains frequency band borders. Frequency
bands are groups of consecutive QMF channels. This means that their
bounds, as provided by f_table_lim, should never exceed MAX_M (maximum
number of QMF channels). c.f. ISO/IEC 14496-3:2001

FAAD2 does not verify this, leading to security issues when
processing files defining f_table_lim with values > MAX_M.

This patch sanitizes the values of f_table_lim so that they can be safely
used as index for Q_M_lim and G_lim arrays.

Fixes CVE-2018-20194.

Upstream commit:
https://github.com/knik0/faad2/commit/6b4a7cde30f2e2cb03e78ef476cc73179cfffda3.patch

--- libfaad/sbr_hfadj.c.orig	2017-07-06 19:16:40.000000000 +0000
+++ libfaad/sbr_hfadj.c
@@ -485,6 +485,12 @@ static void calculate_gain(sbr_info *sbr
             ml1 = sbr->f_table_lim[sbr->bs_limiter_bands][k];
             ml2 = sbr->f_table_lim[sbr->bs_limiter_bands][k+1];
 
+            if (ml1 > MAX_M)
+                ml1 = MAX_M;
+
+            if (ml2 > MAX_M)
+                ml2 = MAX_M;
+
 
             /* calculate the accumulated E_orig and E_curr over the limiter band */
             for (m = ml1; m < ml2; m++)
@@ -949,6 +955,12 @@ static void calculate_gain(sbr_info *sbr
             ml1 = sbr->f_table_lim[sbr->bs_limiter_bands][k];
             ml2 = sbr->f_table_lim[sbr->bs_limiter_bands][k+1];
 
+            if (ml1 > MAX_M)
+                ml1 = MAX_M;
+
+            if (ml2 > MAX_M)
+                ml2 = MAX_M;
+
 
             /* calculate the accumulated E_orig and E_curr over the limiter band */
             for (m = ml1; m < ml2; m++)
@@ -1193,6 +1205,12 @@ static void calculate_gain(sbr_info *sbr
             ml1 = sbr->f_table_lim[sbr->bs_limiter_bands][k];
             ml2 = sbr->f_table_lim[sbr->bs_limiter_bands][k+1];
 
+            if (ml1 > MAX_M)
+                ml1 = MAX_M;
+
+            if (ml2 > MAX_M)
+                ml2 = MAX_M;
+
 
             /* calculate the accumulated E_orig and E_curr over the limiter band */
             for (m = ml1; m < ml2; m++)

$NetBSD: patch-CVE-2018-20362,v 1.1 2019/07/11 09:03:35 nia Exp $

Implicit channel mapping reconfiguration is explicitely forbidden by
ISO/IEC 13818-7:2006 (8.5.3.3). Decoders should be able to detect such
files and reject them. FAAD2 does not perform any kind of checks
regarding this.

This leads to security vulnerabilities when processing crafted AAC
files performing such reconfigurations.

Add checks to decode_sce_lfe and decode_cpe to make sure such
inconsistencies are detected as early as possible.

These checks first read hDecoder->frame: if this is not the first
frame then we make sure that the syntax element at the same position
in the previous frame also had element_id id_syn_ele. If not, return
21 as this is a fatal file structure issue.

This patch addresses CVE-2018-20362 and possibly other related issues.

Upstream commit:
https://github.com/knik0/faad2/commit/466b01d504d7e45f1e9169ac90b3e34ab94aed14.patch

Buffer overflow fix, no CVE, upstream commit:
https://github.com/knik0/faad2/commit/942c3e0aee748ea6fe97cb2c1aa5893225316174.patch

--- libfaad/syntax.c.orig	2017-10-30 17:44:16.000000000 +0000
+++ libfaad/syntax.c
@@ -344,6 +344,12 @@ static void decode_sce_lfe(NeAACDecStruc
        can become 2 when some form of Parametric Stereo coding is used
     */
 
+    if (hDecoder->frame && hDecoder->element_id[hDecoder->fr_ch_ele] != id_syn_ele) {
+        /* element inconsistency */
+        hInfo->error = 21;
+        return;
+    }
+
     /* save the syntax element id */
     hDecoder->element_id[hDecoder->fr_ch_ele] = id_syn_ele;
 
@@ -395,6 +401,12 @@ static void decode_cpe(NeAACDecStruct *h
         return;
     }
 
+    if (hDecoder->frame && hDecoder->element_id[hDecoder->fr_ch_ele] != id_syn_ele) {
+        /* element inconsistency */
+        hInfo->error = 21;
+        return;
+    }
+
     /* save the syntax element id */
     hDecoder->element_id[hDecoder->fr_ch_ele] = id_syn_ele;
 
@@ -2292,6 +2304,8 @@ static uint8_t excluded_channels(bitfile
     while ((drc->additional_excluded_chns[n-1] = faad_get1bit(ld
         DEBUGVAR(1,104,"excluded_channels(): additional_excluded_chns"))) == 1)
     {
+        if (i >= MAX_CHANNELS - num_excl_chan - 7)
+            return n;
         for (i = num_excl_chan; i < num_excl_chan+7; i++)
         {
             drc->exclude_mask[i] = faad_get1bit(ld

$NetBSD: patch-libfaad_bits.c,v 1.1 2019/07/11 09:03:35 nia Exp $

Fix a potential buffer overflow.

Upstream commit:
https://github.com/knik0/faad2/commit/942c3e0aee748ea6fe97cb2c1aa5893225316174.patch

--- libfaad/bits.c.orig	2017-07-06 19:16:40.000000000 +0000
+++ libfaad/bits.c
@@ -167,7 +167,10 @@ void faad_resetbits(bitfile *ld, int bit
     int words = bits >> 5;
     int remainder = bits & 0x1F;
 
-    ld->bytes_left = ld->buffer_size - words*4;
+    if (ld->buffer_size < words * 4)
+        ld->bytes_left = 0;
+    else
+        ld->bytes_left = ld->buffer_size - words*4;
 
     if (ld->bytes_left >= 4)
     {