The problematic loop is the while loop in this snippet from libjbig/jbig.c:jbg_dec_in():

  /* read in DPTABLE */
  if (s->bie_len < 20 + 1728 && 
      (s->options & (JBG_DPON | JBG_DPPRIV | JBG_DPLAST)) ==
      (JBG_DPON | JBG_DPPRIV)) {
    assert(s->bie_len >= 20);
    while (s->bie_len < 20 + 1728 && *cnt < len)
      s->buffer[s->bie_len++ - 20] = data[(*cnt)++];
    if (s->bie_len < 20 + 1728) 
      return JBG_EAGAIN;
    if (!s->dppriv || s->dppriv == jbg_dptable)
      s->dppriv = (char *) checked_malloc(1728, sizeof(char));
    jbg_dppriv2int(s->dppriv, s->buffer);
  }

s->buffer has only 20 bytes, and the loop copies much more than that.