libjulius/src/spsegment.c

説明を見る。

/*
 * Copyright (c) 1991-2011 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2011 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#include <julius/julius.h>


boolean
is_sil(WORD_ID w, RecogProcess *r)
{
  WORD_INFO *winfo;
  HTK_HMM_INFO *hmm;
  int i;

  winfo = r->lm->winfo;
  hmm = r->am->hmminfo;

  /* num of phones should be 1 */
  if (winfo->wlen[w] > 1) return FALSE;

  if (r->pass1.pausemodel) {
    /* has pause model list */
    for(i=0;i<r->pass1.pausemodelnum;i++) {
      if (strmatch(winfo->wseq[w][0]->name, r->pass1.pausemodel[i])) {
        return TRUE;
      }
    }
  } else {
    /* short pause (specified by "-spmodel") */
    if (winfo->wseq[w][0] == hmm->sp) return TRUE;
    
    if (r->lmtype == LM_PROB) {
      /* head/tail sil */
      if (w == winfo->head_silwid || w == winfo->tail_silwid) return TRUE;
    }
  }

  return FALSE;
}

void
mfcc_copy_to_rest_and_shrink(MFCCCalc *mfcc, int start, int end)
{
  int t;

  /* copy rest parameters for next process */
  mfcc->rest_param = new_param();
  memcpy(&(mfcc->rest_param->header), &(mfcc->param->header), sizeof(HTK_Param_Header));
  mfcc->rest_param->samplenum = mfcc->param->samplenum - start;
  mfcc->rest_param->header.samplenum = mfcc->rest_param->samplenum;
  mfcc->rest_param->veclen = mfcc->param->veclen;
  if (param_alloc(mfcc->rest_param, mfcc->rest_param->samplenum, mfcc->rest_param->veclen) == FALSE) {
    j_internal_error("ERROR: segmented: failed to allocate memory for rest param\n");
  }
  /* copy data */
  for(t=start;t<mfcc->param->samplenum;t++) {
    memcpy(mfcc->rest_param->parvec[t-start], mfcc->param->parvec[t], sizeof(VECT) * mfcc->rest_param->veclen);
  }
  
  /* shrink original param */
  /* just shrink the length */
  mfcc->param->samplenum = end;
}

void
mfcc_shrink(MFCCCalc *mfcc, int p)
{
  int t;
  int len;

  if (p > 0) {
    /* copy data */
    for(t=p;t<mfcc->param->samplenum;t++) {
      memcpy(mfcc->param->parvec[t-p], mfcc->param->parvec[t], sizeof(VECT) * mfcc->param->veclen);
    }
    /* shrink original param */
    /* just shrink the length */
    len = mfcc->param->samplenum - p;
    mfcc->param->samplenum = len;
    mfcc->param->header.samplenum = len;
  }
}

boolean
detect_end_of_segment(RecogProcess *r, int time)
{
  FSBeam *d;
  TRELLIS_ATOM *tre;
  LOGPROB maxscore = LOG_ZERO;
  TRELLIS_ATOM *tremax = NULL;
  int count = 0;
  boolean detected = FALSE;
#ifdef SPSEGMENT_NAIST
  MFCCCalc *mfcc;
  WORD_ID wid;
  int j;
  TOKEN2 *tk;
  int startframe;
#endif

  d = &(r->pass1);

#ifdef SPSEGMENT_NAIST

  if (! d->after_trigger) {
    /* we are in the first long pause segment before trigger */

    /* find word end of maximum score from beam status */
    for (j = d->n_start; j <= d->n_end; j++) {
      tk = &(d->tlist[d->tn][d->tindex[d->tn][j]]);
      if (r->wchmm->stend[tk->node] != WORD_INVALID) {
        if (maxscore < tk->score) {
          maxscore = tk->score;
          wid = r->wchmm->stend[tk->node];
        }
      }
    }
    if (maxscore == LOG_ZERO) detected = TRUE;
    else if (is_sil(wid, r)) detected = TRUE;
 
    if (detected) {
      /***********************/
      /* this is noise frame */
      /***********************/

      /* reset trigger duration */
      d->trigger_duration = 0;
      
      /* if noise goes more than a certain frame, shrink the noise area
         to avoid unlimited memory usage */
      if (r->am->mfcc->f > SPSEGMENT_NAIST_AUTOSHRINK_LIMIT) {
        d->want_rewind = TRUE;
        d->rewind_frame = r->am->mfcc->f - r->config->successive.sp_margin;
        d->want_rewind_reprocess = FALSE;
        if (debug2_flag) {
          jlog("DEBUG: pause exceeded %d, rewind\n", SPSEGMENT_NAIST_AUTOSHRINK_LIMIT);
        }
        return FALSE;
      }

      /* keep going */
      d->want_rewind = FALSE;

    } else {
      /************************/
      /* this is speech frame */
      /************************/

      /* increment trigger duration */
      d->trigger_duration++;
      
      /* if not enough duration, not treat as up trigger */
      if (d->trigger_duration < r->config->successive.sp_delay) {
        /* just continue detection */
        return FALSE;
      }

      /***************************/
      /* found speech up-trigger */
      /***************************/
      /* set backstep point */
      if (r->am->mfcc->f < r->config->successive.sp_margin) {
        startframe = 0;
      } else {
        startframe = r->am->mfcc->f - r->config->successive.sp_margin;
      }
      if (debug2_flag) {
        jlog("DEBUG: speech triggered\n");
        jlog("DEBUG: word=[%s] dur=%d\n", r->lm->winfo->woutput[wid], d->trigger_duration);
        jlog("DEBUG: backstep behind %d (from %d to %d) frame and start process\n", r->config->successive.sp_margin, r->am->mfcc->f, startframe);
      }

      /* if the pause segment was short, keep the context of last segment.
         else, reset the context */
      if (r->lmtype == LM_PROB) {
        if (startframe > 0) {
          r->sp_break_last_word = WORD_INVALID;
        }
      }

      /* reset sp duration */
      d->sp_duration = 0;

      /* request the caller to rewind the search to the backstep point and
         re-start with normal search */
      d->want_rewind = TRUE;
      d->rewind_frame = startframe;
      d->want_rewind_reprocess = TRUE;
      /* this will enter to normal search in the next processing */
      d->after_trigger = TRUE;
    }
    /* tell the caller not to segment */
    return FALSE;
  }

#endif /* SPSEGMENT_NAIST */

  /* look for the best trellis word on the given time frame */
  for(tre = r->backtrellis->list; tre != NULL && tre->endtime == time; tre = tre->next) {
    if (maxscore < tre->backscore) {
      maxscore = tre->backscore;
      tremax = tre;
    }
    count++;
  }
  if (tremax == NULL) { /* no word end: possible in the very beggining of input*/
    detected = TRUE;            /* assume it's in the short-pause duration */
  } else if (count > 0) {       /* many words found --- check if maximum is sp */
    if (is_sil(tremax->wid, r)) {
      detected = TRUE;
    }
  }


#ifdef SPSEGMENT_NAIST
  /************************************************************************/
  /************************************************************************/

  /* detected = TRUE if noise frame, or FALSE if speech frame */

  /* sp区間持続チェック */
  /* check sp segment duration */
  if (d->first_sparea) {
    /* we are in the first sp segment */
    if (d->in_sparea && detected) {
      /* sp continues */
      d->sp_duration++;
      /* when sp continues more than -spdur plus -spmargin,
         it means that although a speech trigger has been detected
         by some reason, no actual speech has been found at first. */
      /* in this case we force trigger to end this input */
      if (d->sp_duration > r->config->successive.sp_delay + r->config->successive.sp_margin + r->config->successive.sp_frame_duration) {
        d->in_sparea = FALSE;
        d->first_sparea = FALSE;
        if (debug2_flag) {
          jlog("DEBUG: no valid speech starts, force trigger at %d\n", r->am->mfcc->f);
        }
      }
    } else if (d->in_sparea && !detected) {
      /* found speech frame */
      d->in_sparea = FALSE;
      d->first_sparea = FALSE;
      if (debug2_flag) {
        jlog("DEBUG: speech segment start at %d\n", r->am->mfcc->f);
      }
    }
  } else {
    /* we are either in speech segment, or trailing sp segment */
    if (!d->in_sparea) {
      /* we are in speech segment */
      if (detected) {
        /* detected end of speech segment (begin of sp segment) */
        /* 一時的に開始フレームとしてマーク */
        /* mark this frame as "temporal" begging of short-pause segment */
        d->tmp_sparea_start = time;
#ifdef SP_BREAK_RESUME_WORD_BEGIN
        if (r->lmtype == LM_PROB) {
          /* sp 区間開始時点の最尤単語を保存 */
          /* store the best word in this frame as resuming word */
          d->tmp_sp_break_last_word = tremax ? tremax->wid : WORD_INVALID;
        }
#endif
        d->in_sparea = TRUE;
        d->sp_duration = 1;
      } else {
        /* speech continues */
        /* keep recognizing */
      }
    } else {
      /* we are in trailing sp segment */
      if (detected) {
        /* short pause frame continues */
        d->sp_duration++;
        /* keep word as the "beggining" of next sp segment */
        if (r->lmtype == LM_PROB) {
#ifdef SP_BREAK_RESUME_WORD_BEGIN
          /* if this segment has triggered by (tremax == NULL) (in case the first
             several frame of input), the sp word (to be used as resuming
             word in the next segment) is not yet set. it will be detected here */
          if (d->tmp_sp_break_last_word == WORD_INVALID) {
            if (tremax != NULL) d->tmp_sp_break_last_word = tremax->wid;
          }
#else
          /* resume word at the "end" of sp segment */
          /* simply update the best sp word */
          if (tremax != NULL) d->last_tre_word = tremax->wid;
#endif
        }

        if (d->sp_duration >= r->config->successive.sp_frame_duration) {
          /* silence over, segment the recognition here */
          /* store begging frame of the segment */
          //d->sparea_start = d->tmp_sparea_start;
          r->am->mfcc->sparea_start = time - r->config->successive.sp_frame_duration;
          if (r->lmtype == LM_PROB) {
#ifdef SP_BREAK_RESUME_WORD_BEGIN
            /* resume word = most likely sp word on beginning frame of the segment */
            r->sp_break_last_word = d->tmp_sp_break_last_word;
#else
            /* resume word = most likely sp word on end frame of the segment */
            r->sp_break_last_word = d->last_tre_word;
#endif
          }

          if (debug2_flag) {
            jlog("DEBUG: trailing silence end, end this segment at %d\n", r->am->mfcc->f);
          }
          
          d->after_trigger = FALSE;
          d->trigger_duration = 0;
          d->want_rewind = FALSE;

          /*** segment: [sparea_start - time-1] ***/
          return(TRUE);
        }
        /* else, keep recognition */
      } else {
        /* speech re-triggered */
        /* keep recognition */
        d->in_sparea = FALSE;
      }
    }
  }

  d->want_rewind = FALSE;


#else  /* ~SPSEGMENT_NAIST */
  /************************************************************************/
  /************************************************************************/

  /* sp区間持続チェック */
  /* check sp segment duration */
  if (d->in_sparea && detected) {       /* we are already in sp segment and sp continues */
    d->sp_duration++;           /* increment count */
#ifdef SP_BREAK_RESUME_WORD_BEGIN
    /* resume word at the "beggining" of sp segment */
    /* if this segment has triggered by (tremax == NULL) (in case the first
       several frame of input), the sp word (to be used as resuming
       word in the next segment) is not yet set. it will be detected here */
    if (d->tmp_sp_break_last_word == WORD_INVALID) {
      if (tremax != NULL) d->tmp_sp_break_last_word = tremax->wid;
    }
#else
    /* resume word at the "end" of sp segment */
    /* simply update the best sp word */
    if (tremax != NULL) d->last_tre_word = tremax->wid;
#endif
  }

  /* sp区間開始チェック */
  /* check if sp segment begins at this frame */
  else if (!d->in_sparea && detected) {
    /* 一時的に開始フレームとしてマーク */
    /* mark this frame as "temporal" begging of short-pause segment */
    d->tmp_sparea_start = time;
#ifdef SP_BREAK_RESUME_WORD_BEGIN
    /* sp 区間開始時点の最尤単語を保存 */
    /* store the best word in this frame as resuming word */
    d->tmp_sp_break_last_word = tremax ? tremax->wid : WORD_INVALID;
#endif
    d->in_sparea = TRUE;                /* yes, we are in sp segment */
    d->sp_duration = 1;         /* initialize duration count */
#ifdef SP_BREAK_DEBUG
    jlog("DEBUG: sp start %d\n", time);
#endif /* SP_BREAK_DEBUG */
  }
  
  /* sp 区間終了チェック */
  /* check if sp segment ends at this frame */
  else if (d->in_sparea && !detected) {
    /* (time-1) is end frame of pause segment */
    d->in_sparea = FALSE;               /* we are not in sp segment */
#ifdef SP_BREAK_DEBUG
    jlog("DEBUG: sp end %d\n", time);
#endif /* SP_BREAK_DEBUG */
    /* sp 区間長チェック */
    /* check length of the duration*/
    if (d->sp_duration < r->config->successive.sp_frame_duration) {
      /* 短すぎる: 第１パスを中断せず続行 */
      /* too short segment: not break, continue 1st pass */
#ifdef SP_BREAK_DEBUG
      jlog("DEBUG: too short (%d<%d), ignored\n", d->sp_duration, r->config->successive.sp_frame_duration);
#endif /* SP_BREAK_DEBUG */
    } else if (d->first_sparea) {
      /* 最初のsp区間は silB にあたるので,第１パスを中断せず続行 */
      /* do not break at first sp segment: they are silB */
      d->first_sparea = FALSE;
#ifdef SP_BREAK_DEBUG
      jlog("DEBUG: first silence, ignored\n");
#endif /* SP_BREAK_DEBUG */
    } else {
      /* 区間終了確定, 第１パスを中断して第2パスへ */
      /* break 1st pass */
#ifdef SP_BREAK_DEBUG
      jlog("DEBUG: >> segment [%d..%d]\n", r->am->mfcc->sparea_start, time-1);
#endif /* SP_BREAK_DEBUG */
      /* store begging frame of the segment */
      r->am->mfcc->sparea_start = d->tmp_sparea_start;
#ifdef SP_BREAK_RESUME_WORD_BEGIN
      /* resume word = most likely sp word on beginning frame of the segment */
      r->sp_break_last_word = d->tmp_sp_break_last_word;
#else
      /* resume word = most likely sp word on end frame of the segment */
      r->sp_break_last_word = d->last_tre_word;
#endif

      /*** segment: [sparea_start - time-1] ***/
      return(TRUE);
    }
  }


#endif  /* ~SPSEGMENT_NAIST */

    
#ifdef SP_BREAK_EVAL
  jlog("DEBUG: [%d %d %d]\n", time, count, (detected) ? 50 : 0);
#endif
  return (FALSE);
}

/*******************************************************************/
/* 第１パスセグメント終了処理 (ショートポーズセグメンテーション用) */
/* end of 1st pass for a segment (for short pause segmentation)    */
/*******************************************************************/
void
finalize_segment(Recog *recog)
{
  int spstart;
  RecogProcess *r;
  MFCCCalc *mfcc;
  boolean ok_p;

  /* トレリス始終端における最尤単語を第2パスの始終端単語として格納 */
  /* fix initial/last word hypothesis of the next 2nd pass to the best
     word hypothesis at the first/last frame in backtrellis*/
  for(r=recog->process_list;r;r=r->next) {
    if (!r->live) continue;
    if (r->lmtype == LM_PROB) {
      set_terminal_words(r);
    }
  }

  /* パラメータを, 今第１パスが終了したセグメント区間と残りの区間に分割する. 
     ただし接続部のsp区間部分(sparea_start..len-1)は「のりしろ」として両方に
     コピーする */
  /* Divide input parameter into two: the last segment and the rest.
     The short-pause area (sparea_start..len-1) is considered as "tab",
     copied in both parameters
   */
  /* param[sparea_start..framelen] -> rest_param
     param[0..len-1] -> param
     [sparea_start...len-1] overlapped
  */

  ok_p = FALSE;
  for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
    if (mfcc->segmented) {
      spstart = mfcc->sparea_start;
      ok_p = TRUE;
      break;
    }
  }

  if (ok_p) {
    /* the input was segmented in an instance */
    /* shrink all param the len and store restart parameters in rest_param */
    /* for each mfcc */
    if (verbose_flag) jlog("STAT: segmented: next decoding will restart from %d\n", spstart);

    for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
      if (verbose_flag) jlog("STAT: MFCC%02d: segmented: processed length=%d\n", mfcc->id, mfcc->last_time);

      /* copy the rest to mfcc->rest_param and shrink mfcc->param */
      mfcc_copy_to_rest_and_shrink(mfcc, spstart, mfcc->last_time);
    }

    /* reset last_word info */
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      r->sp_break_last_nword_allow_override = TRUE;
    }
    
  } else {
    
    /* last segment is on end of input: no rest parameter */
    for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
      mfcc->rest_param = NULL;
    }
    /* reset last_word info */
    for(r=recog->process_list;r;r=r->next) {
      if (!r->live) continue;
      r->sp_break_2_begin_word = WORD_INVALID;
      r->sp_break_last_word = WORD_INVALID;
      r->sp_break_last_nword = WORD_INVALID;
      r->sp_break_last_nword_allow_override = FALSE;
    }
  }
}

#ifdef BACKEND_VAD

void
spsegment_init(Recog *recog)
{
  RecogProcess *p;
  /* at first time, recognition does not start yet */
#ifdef SPSEGMENT_NAIST
  for(p=recog->process_list;p;p=p->next) {
    p->pass1.after_trigger = FALSE;
    p->pass1.trigger_duration = 0;
  }
#endif
#ifdef GMM_VAD
  if (recog->gmm) {
    recog->gc->after_trigger = FALSE;
    recog->gc->duration = 0;
  }
#endif
  recog->triggered = FALSE;
}

boolean
spsegment_trigger_sync(Recog *recog)
{
  RecogProcess *p;
  boolean ok_p;

  ok_p = FALSE;
  if (recog->jconf->decodeopt.segment) {
#ifdef SPSEGMENT_NAIST
    for(p = recog->process_list; p; p = p->next) {
      if (!p->live) continue;
      if (p->pass1.after_trigger) {
        ok_p = TRUE;
        break;
      }
    }
#endif
#ifdef GMM_VAD
    if (recog->gmm) {
      if (recog->gc->after_trigger) {
        ok_p = TRUE;
      }
    }
#endif
  }
  if (ok_p) {
    /* up trigger detected */
#ifdef SPSEGMENT_NAIST
    for(p = recog->process_list; p; p = p->next) {
      if (!p->live) continue;
      p->pass1.after_trigger = TRUE;
    }
#endif
#ifdef GMM_VAD
    if (recog->gmm) {
      recog->gc->after_trigger = TRUE;
    }
#endif
  }
  
  return ok_p;
}

#endif /* BACKEND_VAD */

boolean
spsegment_need_restart(Recog *recog, int *rf_ret, boolean *repro_ret)
{
#ifdef SPSEGMENT_NAIST
  RecogProcess *p;
#endif
  boolean ok_p;
  int rewind_frame = 0;
  boolean reprocess = FALSE;

  ok_p = FALSE;
  if (recog->jconf->decodeopt.segment) {
#ifdef SPSEGMENT_NAIST
    /* check for rewind request from each process */
    for(p = recog->process_list; p; p = p->next) {
      if (!p->live) continue;
      if (p->pass1.want_rewind) {
        p->pass1.want_rewind = FALSE;
        rewind_frame = p->pass1.rewind_frame;
        reprocess = p->pass1.want_rewind_reprocess;
        ok_p = TRUE;
        break;
      }
    }
#endif /* SPSEGMENT_NAIST */
#ifdef GMM_VAD
    if (recog->gmm) {
      if (recog->gc->want_rewind) {
        recog->gc->want_rewind = FALSE;
#ifdef SPSEGMENT_NAIST
        /* set to earlier one */
        if (rewind_frame > recog->gc->rewind_frame) rewind_frame = recog->gc->rewind_frame;
#else
        rewind_frame = recog->gc->rewind_frame;
#endif
        reprocess = recog->gc->want_rewind_reprocess;
        ok_p = TRUE;
      }
    }
#endif
    *rf_ret = rewind_frame;
    *repro_ret = reprocess;
  }

  return(ok_p);
}

void
spsegment_restart_mfccs(Recog *recog, int rewind_frame, boolean reprocess)
{
  MFCCCalc *mfcc;

  for (mfcc = recog->mfcclist; mfcc; mfcc = mfcc->next) {
    if (!mfcc->valid) continue;
    /* set last segmented time */
    mfcc->last_time = mfcc->f - 1;
    /* reset frame pointers */
    if (reprocess) {
      /* set all mfcc to initial point for re-process the whole frames */
      mfcc->f = -1;
    } else {
      /* just bring back to the new last point after shrink */
      mfcc->f -= rewind_frame;
    }
    /* shrink the current mfcc */
    mfcc_shrink(mfcc, rewind_frame);
  }
}

/* end of file */