The problematic loop is the while loop in this snippet from libjbig/jbig.c:jbg_dec_in():
/* read in DPTABLE */
if (s->bie_len < 20 + 1728 &&
(s->options & (JBG_DPON | JBG_DPPRIV | JBG_DPLAST)) ==
(JBG_DPON | JBG_DPPRIV)) {
assert(s->bie_len >= 20);
while (s->bie_len < 20 + 1728 && *cnt < len)
s->buffer[s->bie_len++ - 20] = data[(*cnt)++];
if (s->bie_len < 20 + 1728)
return JBG_EAGAIN;
if (!s->dppriv || s->dppriv == jbg_dptable)
s->dppriv = (char *) checked_malloc(1728, sizeof(char));
jbg_dppriv2int(s->dppriv, s->buffer);
}
s->buffer has only 20 bytes, and the loop copies much more than that.