00001
00116
00117
00118
00119
00120
00121
00122
00123 #include <julius.h>
00124
00125 #undef RDEBUG
00126
00127
00128 static HTK_Param *param = NULL;
00129 static float *bf;
00130 static DeltaBuf *db;
00131 static DeltaBuf *ab;
00132 static VECT *tmpmfcc;
00133 static int maxframelen;
00134 static int last_time;
00135
00136 static boolean last_is_segmented;
00137 #ifdef SP_BREAK_CURRENT_FRAME
00138 static SP16 *rest_Speech = NULL;
00139 static int rest_alloc_len = 0;
00140 static int rest_len;
00141 #endif
00142
00143 static int f_raw;
00144 static int f;
00145 static SP16 *window;
00146 static int windowlen;
00147 static int windownum;
00148
00149
00150
00151
00152
00164 static void
00165 init_param()
00166 {
00167
00168
00169 param = new_param();
00170
00171
00172 param->header.samptype = F_MFCC;
00173 if (para.delta) param->header.samptype |= F_DELTA;
00174 if (para.acc) param->header.samptype |= F_ACCL;
00175 if (para.energy) param->header.samptype |= F_ENERGY;
00176 if (para.c0) param->header.samptype |= F_ZEROTH;
00177 if (para.absesup) param->header.samptype |= F_ENERGY_SUP;
00178 if (para.cmn) param->header.samptype |= F_CEPNORM;
00179
00180 param->header.wshift = para.smp_period * para.frameshift;
00181 param->header.sampsize = para.veclen * sizeof(VECT);
00182 param->veclen = para.veclen;
00183
00184
00185
00186 param->parvec = (VECT **)mymalloc(sizeof(VECT *) * maxframelen);
00187
00188
00189
00190
00191
00192
00193
00194
00195 }
00196
00206 void
00207 RealTimeInit()
00208 {
00209
00210
00211 if (ssload_filename && ssbuf == NULL) {
00212 if ((ssbuf = new_SS_load_from_file(ssload_filename, &sslen)) == NULL) {
00213 j_error("Error: failed to read \"%s\"\n", ssload_filename);
00214 }
00215 }
00216
00217
00218 WMP_init(para, &bf, ssbuf, sslen);
00219
00220
00221 if (para.energy && para.enormal) energy_max_init();
00222
00223
00224 if (para.delta) db = WMP_deltabuf_new(para.baselen, para.delWin);
00225 if (para.acc) ab = WMP_deltabuf_new(para.baselen * 2, para.accWin);
00226
00227
00228 tmpmfcc = (VECT *)mymalloc(sizeof(VECT) * para.vecbuflen);
00229
00230
00231 maxframelen = MAXSPEECHLEN / para.frameshift;
00232
00233
00234 windowlen = para.framesize + 1;
00235
00236
00237 window = mymalloc(sizeof(SP16) * windowlen);
00238
00239
00240 if (para.cmn) CMN_realtime_init(para.mfcc_dim, cmn_map_weight);
00241
00242
00243 if (cmnload_filename) {
00244 if (para.cmn) {
00245 if ((cmn_loaded = CMN_load_from_file(cmnload_filename, para.mfcc_dim))== FALSE) {
00246 j_printf("Warning: failed to read cepstral mean from \"%s\"\n", cmnload_filename);
00247 }
00248 } else {
00249 j_printf("Warning: CMN not required, file \"%s\" ignored\n", cmnload_filename);
00250 }
00251 }
00252 }
00253
00254
00255
00265 void
00266 RealTimePipeLinePrepare()
00267 {
00268
00269
00270 init_param();
00271
00272
00273 if (para.energy && para.enormal) energy_max_prepare(¶);
00274
00275
00276 if (para.delta) WMP_deltabuf_prepare(db);
00277 if (para.acc) WMP_deltabuf_prepare(ab);
00278
00279
00280
00281
00282 outprob_prepare(maxframelen);
00283
00284
00285 if (!check_param_coherence(hmminfo, param)) {
00286 j_error("parameter type not match?\n");
00287 }
00288
00289
00290 f_raw = 0;
00291 f = 0;
00292 windownum = 0;
00293
00294
00295
00296 if (para.cmn) CMN_realtime_prepare();
00297
00298
00299
00300 if (record_dirname != NULL) {
00301 record_sample_open();
00302 }
00303
00304 #ifdef VISUALIZE
00305
00306
00307
00308 speechlen = 0;
00309 #endif
00310 }
00311
00350 int
00351 RealTimePipeLine(SP16 *Speech, int nowlen)
00352 {
00353 int i, now;
00354 boolean ret;
00355 LOGPROB f;
00356
00357
00358
00359
00360
00361
00362 now = 0;
00363
00364
00365
00366 last_is_segmented = FALSE;
00367
00368
00369
00370 if (f_raw == 0) status_recstart();
00371
00372 #ifdef RDEBUG
00373 printf("got %d samples\n", nowlen);
00374 #endif
00375
00376
00377
00378 if (record_dirname != NULL) {
00379 record_sample_write(Speech, nowlen);
00380 }
00381
00382 #ifdef VISUALIZE
00383
00384 adin_cut_callback_store_buffer(Speech, nowlen);
00385 #endif
00386
00387 while (now < nowlen) {
00388
00389
00390 if (f_raw >= maxframelen) return(1);
00391
00392
00393 for(i = min(windowlen - windownum, nowlen - now); i > 0 ; i--)
00394 window[windownum++] = (float) Speech[now++];
00395
00396
00397
00398
00399 if (windownum < windowlen) break;
00400 #ifdef RDEBUG
00401
00402
00403
00404 #endif
00405
00406
00407
00408 for (i=0; i < windowlen; i++) {
00409 bf[i+1] = (float) window[i];
00410 }
00411 WMP_calc(tmpmfcc, bf, para, ssbuf);
00412
00413 if (para.energy && para.enormal) {
00414
00415
00416
00417
00418
00419
00420
00421
00422 tmpmfcc[para.baselen-1] = energy_max_normalize(tmpmfcc[para.baselen-1], ¶);
00423 }
00424
00425 if (para.delta) {
00426
00427
00428 ret = WMP_deltabuf_proceed(db, tmpmfcc);
00429 #ifdef RDEBUG
00430 printf("DeltaBuf: ret=%d, status=", ret);
00431 for(i=0;i<db->len;i++) {
00432 printf("%d", db->is_on[i]);
00433 }
00434 printf(", nextstore=%d\n", db->store);
00435 #endif
00436
00437
00438
00439 if (! ret) {
00440 goto next_input;
00441 }
00442
00443
00444 memcpy(tmpmfcc, db->vec, sizeof(VECT) * para.baselen * 2);
00445 }
00446
00447 if (para.acc) {
00448
00449
00450
00451
00452 ret = WMP_deltabuf_proceed(ab, tmpmfcc);
00453 #ifdef RDEBUG
00454 printf("AccelBuf: ret=%d, status=", ret);
00455 for(i=0;i<ab->len;i++) {
00456 printf("%d", ab->is_on[i]);
00457 }
00458 printf(", nextstore=%d\n", ab->store);
00459 #endif
00460
00461
00462
00463 if (! ret) {
00464 goto next_input;
00465 }
00466
00467
00468
00469
00470
00471
00472 memcpy(tmpmfcc, ab->vec, sizeof(VECT) * para.baselen * 2);
00473 memcpy(&(tmpmfcc[para.baselen*2]), &(ab->vec[para.baselen*3]), sizeof(VECT) * para.baselen);
00474 }
00475
00476 if (para.delta && (para.energy || para.c0) && para.absesup) {
00477
00478
00479 memmove(&(tmpmfcc[para.baselen-1]), &(tmpmfcc[para.baselen]), sizeof(VECT) * (para.vecbuflen - para.baselen));
00480 }
00481
00482
00483
00484
00485 #ifdef RDEBUG
00486 printf("DeltaBuf: got frame %d\n", f_raw);
00487 #endif
00488
00489
00490 if (para.cmn) CMN_realtime(tmpmfcc, para.mfcc_dim);
00491
00492
00493
00494 param->parvec[f_raw] = (VECT *)mymalloc(sizeof(VECT) * param->veclen);
00495 memcpy(param->parvec[f_raw], tmpmfcc, sizeof(VECT) * param->veclen);
00496 f = f_raw;
00497
00498
00499
00500
00501
00502 if (f == 0) {
00503
00504
00505 get_back_trellis_init(param, wchmm, &backtrellis);
00506 }
00507 #ifndef MULTIPATH_VERSION
00508 if (f != 0) {
00509 #endif
00510
00511
00512 if (get_back_trellis_proceed(f, param, wchmm, &backtrellis
00513 #ifdef MULTIPATH_VERSION
00514 ,FALSE
00515 #endif
00516 ) == FALSE) {
00517
00518
00519
00520
00521
00522
00523
00524
00525
00526
00527
00528
00529
00530
00531
00532 last_is_segmented = TRUE;
00533
00534
00535 last_time = f-1;
00536 #ifdef SP_BREAK_CURRENT_FRAME
00537
00538
00539
00540
00541 param->header.samplenum = f_raw+1;
00542 param->samplenum = f_raw+1;
00543 rest_len = nowlen - now;
00544 if (rest_len > 0) {
00545
00546 if (rest_Speech == NULL) {
00547 rest_alloc_len = rest_len;
00548 rest_Speech = (SP16 *)mymalloc(sizeof(SP16)*rest_alloc_len);
00549 } else if (rest_alloc_len < rest_len) {
00550 rest_alloc_len = rest_len;
00551 rest_Speech = (SP16 *)myrealloc(rest_Speech, sizeof(SP16)*rest_alloc_len);
00552 }
00553 memcpy(rest_Speech, &(Speech[now]), sizeof(SP16) * rest_len);
00554 }
00555 #else
00556
00557
00558 param->header.samplenum = f;
00559 param->samplenum = f;
00560 #endif
00561
00562
00563 return(1);
00564 }
00565 #ifndef MULTIPATH_VERSION
00566 }
00567 #endif
00568
00569
00570 f_raw++;
00571
00572 next_input:
00573
00574
00575
00576 memmove(window, &(window[para.frameshift]), sizeof(SP16) * (windowlen - para.frameshift));
00577 windownum -= para.frameshift;
00578 }
00579
00580
00581
00582
00583
00584 return(0);
00585 }
00586
00587 #ifdef SP_BREAK_CURRENT_FRAME
00588
00622 int
00623 RealTimeResume()
00624 {
00625 int t;
00626
00627
00628
00629 param = rest_param;
00630
00631
00632
00633 outprob_prepare(maxframelen);
00634 param->parvec = (VECT **)myrealloc(param->parvec, sizeof(VECT *) * maxframelen);
00635
00636
00637 f_raw = param->samplenum - 1;
00638
00639
00640
00641 if (f_raw >= 0) {
00642 f = f_raw;
00643 #ifdef RDEBUG
00644 printf("Resume: f=%d,f_raw=%d\n", f, f_raw);
00645 #endif
00646 get_back_trellis_init(param, wchmm, &backtrellis);
00647 for (t=
00648 #ifdef MULTIPATH_VERSION
00649 0
00650 #else
00651 1
00652 #endif
00653 ;t<=f;t++) {
00654 if (get_back_trellis_proceed(t, param, wchmm, &backtrellis
00655 #ifdef MULTIPATH_VERSION
00656 ,FALSE
00657 #endif
00658 ) == FALSE) {
00659
00660 last_is_segmented = TRUE;
00661 last_time = t-1;
00662 return(1);
00663 }
00664 }
00665 }
00666
00667 f_raw++;
00668 f = f_raw;
00669
00670
00671
00672 memmove(window, &(window[para.frameshift]), sizeof(SP16) * (windowlen - para.frameshift));
00673 windownum -= para.frameshift;
00674
00675
00676
00677
00678
00679 if (rest_len > 0) {
00680 #ifdef RDEBUG
00681 printf("Resume: rest %d samples\n", rest_len);
00682 #endif
00683 return(RealTimePipeLine(rest_Speech, rest_len));
00684 }
00685
00686
00687
00688 return 0;
00689 }
00690 #endif
00691
00692
00693
00694
00711 HTK_Param *
00712 RealTimeParam(LOGPROB *backmax)
00713 {
00714 boolean ret1, ret2;
00715
00716 if (last_is_segmented) {
00717
00718
00719
00720
00721
00722
00723 *backmax = finalize_1st_pass(&backtrellis, winfo, last_time);
00724 #ifdef SP_BREAK_CURRENT_FRAME
00725 finalize_segment(&backtrellis, param, last_time);
00726 #endif
00727
00728
00729 return(param);
00730 }
00731
00732
00733
00734
00735 if (para.delta || para.acc) {
00736
00737
00738 while(f_raw < maxframelen) {
00739
00740
00741 ret1 = WMP_deltabuf_flush(db);
00742 #ifdef RDEBUG
00743 {
00744 int i;
00745 printf("DeltaBufLast: ret=%d, status=", ret1);
00746 for(i=0;i<db->len;i++) {
00747 printf("%d", db->is_on[i]);
00748 }
00749 printf(", nextstore=%d\n", db->store);
00750 }
00751 #endif
00752 if (ret1) {
00753
00754 if (para.energy && para.absesup) {
00755 memcpy(tmpmfcc, db->vec, sizeof(VECT) * (para.baselen - 1));
00756 memcpy(&(tmpmfcc[para.baselen-1]), &(db->vec[para.baselen]), sizeof(VECT) * para.baselen);
00757 } else {
00758 memcpy(tmpmfcc, db->vec, sizeof(VECT) * para.baselen * 2);
00759 }
00760 if (para.acc) {
00761
00762 ret2 = WMP_deltabuf_proceed(ab, tmpmfcc);
00763 #ifdef RDEBUG
00764 printf("AccelBuf: ret=%d, status=", ret2);
00765 for(i=0;i<ab->len;i++) {
00766 printf("%d", ab->is_on[i]);
00767 }
00768 printf(", nextstore=%d\n", ab->store);
00769 #endif
00770 if (ret2) {
00771
00772 memcpy(tmpmfcc, ab->vec, sizeof(VECT) * (para.veclen - para.baselen));
00773 memcpy(&(tmpmfcc[para.veclen - para.baselen]), &(ab->vec[para.veclen - para.baselen]), sizeof(VECT) * para.baselen);
00774 } else {
00775
00776
00777 continue;
00778 }
00779 }
00780 } else {
00781
00782 if (para.acc) {
00783
00784 ret2 = WMP_deltabuf_flush(ab);
00785 #ifdef RDEBUG
00786 printf("AccelBuf: ret=%d, status=", ret2);
00787 for(i=0;i<ab->len;i++) {
00788 printf("%d", ab->is_on[i]);
00789 }
00790 printf(", nextstore=%d\n", ab->store);
00791 #endif
00792 if (ret2) {
00793
00794 memcpy(tmpmfcc, ab->vec, sizeof(VECT) * (para.veclen - para.baselen));
00795 memcpy(&(tmpmfcc[para.veclen - para.baselen]), &(ab->vec[para.veclen - para.baselen]), sizeof(VECT) * para.baselen);
00796 } else {
00797
00798 break;
00799 }
00800 } else {
00801
00802 break;
00803 }
00804 }
00805 if(para.cmn) CMN_realtime(tmpmfcc, para.mfcc_dim);
00806 param->parvec[f_raw] = (VECT *)mymalloc(sizeof(VECT) * param->veclen);
00807 memcpy(param->parvec[f_raw], tmpmfcc, sizeof(VECT) * param->veclen);
00808 f = f_raw;
00809 if (f == 0) {
00810 get_back_trellis_init(param, wchmm, &backtrellis);
00811 }
00812 #ifdef MULTIPATH_VERSION
00813 get_back_trellis_proceed(f, param, wchmm, &backtrellis, FALSE);
00814 #else
00815 if (f != 0) {
00816 get_back_trellis_proceed(f, param, wchmm, &backtrellis);
00817 }
00818 #endif
00819 f_raw++;
00820 }
00821 }
00822
00823
00824
00825 param->header.samplenum = f_raw;
00826 param->samplenum = f_raw;
00827
00828
00829
00830
00831 if (f_raw == 0) {
00832 j_printf("Error: too short input to compute delta coef! (%d frames)\n", f_raw);
00833 *backmax = finalize_1st_pass(&backtrellis, winfo, param->samplenum);
00834 } else {
00835
00836
00837 get_back_trellis_end(param, wchmm, &backtrellis);
00838 *backmax = finalize_1st_pass(&backtrellis, winfo, param->samplenum);
00839 #ifdef SP_BREAK_CURRENT_FRAME
00840 finalize_segment(&backtrellis, param, param->samplenum);
00841 #endif
00842 }
00843
00844
00845
00846 return(param);
00847 }
00848
00861 void
00862 RealTimeCMNUpdate(HTK_Param *param)
00863 {
00864 float mseclen;
00865 boolean cmn_update_p;
00866
00867
00868 if(para.cmn) {
00869 if (cmn_update) {
00870 cmn_update_p = TRUE;
00871 if (rejectshortlen > 0) {
00872
00873 mseclen = (float)param->samplenum * (float)para.smp_period * (float)para.frameshift / 10000.0;
00874 if (mseclen < rejectshortlen) {
00875 cmn_update_p = FALSE;
00876 }
00877 }
00878 if (gmm_reject_cmn_string != NULL) {
00879
00880 if(! gmm_valid_input()) {
00881 cmn_update_p = FALSE;
00882 }
00883 }
00884 if (cmn_update_p) {
00885
00886 CMN_realtime_update();
00887 } else {
00888
00889 j_printf("CMN not updated\n");
00890 }
00891 }
00892
00893 if (cmnsave_filename) {
00894 if (CMN_save_to_file(cmnsave_filename) == FALSE) {
00895 j_printf("Warning: failed to save cmn data to \"%s\"\n", cmnsave_filename);
00896 }
00897 }
00898 }
00899 }
00900
00909 void
00910 RealTimeTerminate()
00911 {
00912 param->header.samplenum = f_raw;
00913 param->samplenum = f_raw;
00914
00915
00916
00917
00918 if (f_raw == 0) {
00919 finalize_1st_pass(&backtrellis, winfo, param->samplenum);
00920 } else {
00921
00922
00923 status_recend();
00924 get_back_trellis_end(param, wchmm, &backtrellis);
00925 finalize_1st_pass(&backtrellis, winfo, param->samplenum);
00926 #ifdef SP_BREAK_CURRENT_FRAME
00927 finalize_segment(&backtrellis, param, param->samplenum);
00928 #endif
00929 }
00930
00931
00932 free_param(param);
00933 }