libsent/src/ngram/ngram_write_bin.c

説明を見る。

/*
 * Copyright (c) 1991-2006 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2006 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#include <sent/stddefs.h>
#include <sent/ngram2.h>

static boolean need_swap; 

static void
wrt(FILE *fp, void *buf, size_t unitbyte, int unitnum)
{
  if (need_swap == TRUE && unitbyte != 1) {
    swap_bytes((char *)buf, unitbyte, unitnum);
  }
  if (myfwrite(buf, unitbyte, unitnum, fp) < (size_t)unitnum) {
    perror("write_ngram_bin: wrt");
    j_error("write failed\n");
  }
  if (need_swap == TRUE && unitbyte != 1) {
    swap_bytes((char *)buf, unitbyte, unitnum);
  }
}

static void
write_header(FILE *fp, char *str, int version)
{
  char buf[BINGRAM_HDSIZE];
  int i, totallen;
  for(i=0;i<BINGRAM_HDSIZE;i++) buf[i] = EOF;
  switch(version) {
  case 4:
    totallen = strlen(BINGRAM_IDSTR_V4) + 1 + strlen(BINGRAM_SIZESTR_HEAD) + strlen(BINGRAM_SIZESTR_BODY) + 1 + strlen(BINGRAM_BYTEORDER_HEAD) + strlen(BINGRAM_NATURAL_BYTEORDER) + 1 + strlen(str);
    break;
  case 3:
    totallen = strlen(BINGRAM_IDSTR) + 1 + strlen(BINGRAM_SIZESTR_HEAD) + strlen(BINGRAM_SIZESTR_BODY) + 1 + strlen(str);
    break;
  }
  if (totallen >= BINGRAM_HDSIZE) {
    j_printerr("Warning: user header too long, last will be truncated\n");
    i = strlen(str) - (totallen - BINGRAM_HDSIZE);
    str[i] = '\0';
  }
  switch(version) {
  case 4:
    sprintf(buf, "%s\n%s%s %s%s\n%s", BINGRAM_IDSTR_V4, BINGRAM_SIZESTR_HEAD, BINGRAM_SIZESTR_BODY, BINGRAM_BYTEORDER_HEAD, BINGRAM_NATURAL_BYTEORDER, str);
    break;
  case 3:
    sprintf(buf, "%s\n%s%s\n%s", BINGRAM_IDSTR, BINGRAM_SIZESTR_HEAD, BINGRAM_SIZESTR_BODY, str);
    break;
  }
  wrt(fp, buf, 1, BINGRAM_HDSIZE);
}

boolean
ngram_write_bin(FILE *fp, NGRAM_INFO *ndata, char *headerstr)
{
  int i,n,len;

  /* write initial header */
  write_header(fp, headerstr, ndata->version);

  /* set swap requirement */
  if (ndata->version == 4) {
    need_swap = FALSE;
  } else {
#ifdef WORDS_BIGENDIAN
    need_swap = FALSE;
#else
    need_swap = TRUE;
#endif
  } 

  /* write total info */
  for(n=0;n<MAX_N;n++) {
    wrt(fp, &(ndata->ngram_num[n]), sizeof(NNID), 1);
    /*j_printf("ngram %d=%d\n",n+1,ndata->ngram_num[n]);*/
  }
  if (ndata->version == 4) {
    wrt(fp, &(ndata->bigram_bo_num), sizeof(NNID), 1);
  }
  j_printf("wrote total info\n");
  /* unk_*, isopen, max_word_num are set after read, so need not save */

  /* write wname */
  len = 0;
  for(i=0;i<ndata->ngram_num[0];i++) {
    len += strlen(ndata->wname[i]) + 1;
  }
  wrt(fp, &len, sizeof(int), 1);
  for(i=0;i<ndata->ngram_num[0];i++) {
    wrt(fp, ndata->wname[i], 1, strlen(ndata->wname[i]) + 1); /* include \0 */
  }
  j_printf("wrote wnames (%d bytes)\n", len + sizeof(int));
  
  /* write 1-gram */
  wrt(fp, ndata->p, sizeof(LOGPROB), ndata->ngram_num[0]);
  wrt(fp, ndata->bo_wt_lr, sizeof(LOGPROB), ndata->ngram_num[0]);
  wrt(fp, ndata->bo_wt_rl, sizeof(LOGPROB), ndata->ngram_num[0]);
  wrt(fp, ndata->n2_bgn, sizeof(NNID), ndata->ngram_num[0]);
  wrt(fp, ndata->n2_num, sizeof(WORD_ID), ndata->ngram_num[0]);
  j_printf("wrote 1-gram (%d KB)\n",
           ((sizeof(LOGPROB)*3 + sizeof(NNID) + sizeof(WORD_ID)) * ndata->ngram_num[0]) / 1024);
  
  /* write 2-gram*/
  wrt(fp, ndata->n2tonid, sizeof(WORD_ID), ndata->ngram_num[1]);
  wrt(fp, ndata->p_lr, sizeof(LOGPROB), ndata->ngram_num[1]);
  wrt(fp, ndata->p_rl, sizeof(LOGPROB), ndata->ngram_num[1]);
  switch (ndata->version) {
  case 4:
    wrt(fp, ndata->n2bo_upper, sizeof(NNID_UPPER), ndata->ngram_num[1]);
    wrt(fp, ndata->n2bo_lower, sizeof(NNID_LOWER), ndata->ngram_num[1]);
    wrt(fp, ndata->bo_wt_rrl, sizeof(LOGPROB), ndata->bigram_bo_num);
    wrt(fp, ndata->n3_bgn_upper, sizeof(NNID_UPPER), ndata->bigram_bo_num);
    wrt(fp, ndata->n3_bgn_lower, sizeof(NNID_LOWER), ndata->bigram_bo_num);
    wrt(fp, ndata->n3_num, sizeof(WORD_ID), ndata->bigram_bo_num);
    j_printf("wrote 2-gram (%d KB)\n",
             ((sizeof(LOGPROB)*2 + sizeof(NNID_UPPER) + sizeof(NNID_LOWER) + sizeof(WORD_ID)) * ndata->ngram_num[1] + (sizeof(LOGPROB) + sizeof(NNID_UPPER) + sizeof(NNID_LOWER) + sizeof(WORD_ID)) * ndata->bigram_bo_num) / 1024);
    break;
  case 3:
    wrt(fp, ndata->bo_wt_rrl, sizeof(LOGPROB), ndata->ngram_num[1]);
    wrt(fp, ndata->n3_bgn, sizeof(NNID), ndata->ngram_num[1]);
    wrt(fp, ndata->n3_num, sizeof(WORD_ID), ndata->ngram_num[1]);
    j_printf("wrote 2-gram (%d KB)\n",
             ((sizeof(LOGPROB)*3 + sizeof(NNID) + sizeof(WORD_ID)*2) * ndata->ngram_num[1]) / 1024);
    break;
  }
  

  /* write 3-gram*/
  wrt(fp, ndata->n3tonid, sizeof(WORD_ID), ndata->ngram_num[2]);
  wrt(fp, ndata->p_rrl, sizeof(LOGPROB), ndata->ngram_num[2]);
  j_printf("wrote 3-gram (%d KB)\n",
           ((sizeof(LOGPROB) + sizeof(WORD_ID)) * ndata->ngram_num[2]) / 1024);

  return TRUE;
}

---