libsent/include/sent/ngram2.h

説明を見る。

/*
 * Copyright (c) 1991-2011 Kawahara Lab., Kyoto University
 * Copyright (c) 2000-2005 Shikano Lab., Nara Institute of Science and Technology
 * Copyright (c) 2005-2011 Julius project team, Nagoya Institute of Technology
 * All rights reserved
 */

#ifndef __SENT_NGRAM2_H__
#define __SENT_NGRAM2_H__

#include <sent/stddefs.h>
#include <sent/ptree.h>

typedef unsigned int NNID;            
#define NNID_INVALID 0xffffffff  ///< Value to indicate no id (full)
#define NNID_MAX 0xfffffffe     ///< Value of maximum value (full)

typedef unsigned char NNID_UPPER; 
typedef unsigned short NNID_LOWER; 
#define NNID_INVALID_UPPER 255  ///< Value to indicate no id at NNID_UPPER (24bit)
#define NNID_MAX_24 16711679        ///< Allowed maximum number of id (255*65536-1) (24bit)

#define BEGIN_WORD_DEFAULT "<s>"

#define END_WORD_DEFAULT "</s>"

#define UNK_WORD_DEFAULT "<unk>"
#define UNK_WORD_DEFAULT2 "<UNK>"

#define UNK_WORD_MAXLEN 30

typedef struct {
  NNID totalnum;                
  boolean is24bit;              
  NNID bgnlistlen;              
  NNID_UPPER *bgn_upper;        
  NNID_LOWER *bgn_lower;        
  NNID *bgn;                    
  WORD_ID *num;         

  WORD_ID *nnid2wid;            
  LOGPROB *prob;                

  NNID context_num;             
  LOGPROB *bo_wt;               
  boolean ct_compaction;        
  NNID_UPPER *nnid2ctid_upper;  
  NNID_LOWER *nnid2ctid_lower;  

} NGRAM_TUPLE_INFO;

typedef struct __ngram_info__ {
  int n;                        
  int dir;                      
  boolean from_bin;             
  boolean bigram_index_reversed;                
  boolean bos_eos_swap;         
  WORD_ID max_word_num;         
  char **wname;                 
  PATNODE *root;                
  WORD_ID unk_id;               
  int unk_num;                  
  LOGPROB unk_num_log;          
  boolean isopen;               

  NGRAM_TUPLE_INFO *d;  

  /* for pass1 */
  LOGPROB *bo_wt_1;             
  LOGPROB *p_2;                 
  LOGPROB (*bigram_prob)(struct __ngram_info__ *, WORD_ID, WORD_ID); 

  BMALLOC_BASE *mroot;          

} NGRAM_INFO;


/* Definitions for binary N-gram */

#define BINGRAM_IDSTR "julius_bingram_v3"

#define BINGRAM_IDSTR_V4 "julius_bingram_v4"

#define BINGRAM_IDSTR_V5 "julius_bingram_v5"

#define BINGRAM_HDSIZE 512

#define BINGRAM_SIZESTR_HEAD "word="

#define BINGRAM_SIZESTR_BODY_4BYTE "4byte(int)"

#define BINGRAM_SIZESTR_BODY_2BYTE "2byte(unsigned short)"
#ifdef WORDS_INT
#define BINGRAM_SIZESTR_BODY BINGRAM_SIZESTR_BODY_4BYTE
#else
#define BINGRAM_SIZESTR_BODY BINGRAM_SIZESTR_BODY_2BYTE
#endif

#define BINGRAM_BYTEORDER_HEAD "byteorder="

#ifdef WORDS_BIGENDIAN
#define BINGRAM_NATURAL_BYTEORDER "BE"
#else
#define BINGRAM_NATURAL_BYTEORDER "LE"
#endif


#ifdef __cplusplus
extern "C" {
#endif

/* function declaration */
NNID search_ngram(NGRAM_INFO *ndata, int n, WORD_ID *w);
LOGPROB ngram_prob(NGRAM_INFO *ndata, int n, WORD_ID *w);
LOGPROB uni_prob(NGRAM_INFO *ndata, WORD_ID w);
LOGPROB bi_prob(NGRAM_INFO *ndata, WORD_ID w1, WORD_ID w2);
void bi_prob_func_set(NGRAM_INFO *ndata);

boolean ngram_read_arpa(FILE *fp, NGRAM_INFO *ndata, boolean addition);
boolean ngram_read_bin(FILE *fp, NGRAM_INFO *ndata);
boolean ngram_write_bin(FILE *fp, NGRAM_INFO *ndata, char *header_str);

boolean ngram_compact_context(NGRAM_INFO *ndata, int n);

void ngram_make_lookup_tree(NGRAM_INFO *ndata);
WORD_ID ngram_lookup_word(NGRAM_INFO *ndata, char *wordstr);
WORD_ID make_ngram_ref(NGRAM_INFO *, char *);

NGRAM_INFO *ngram_info_new();
void ngram_info_free(NGRAM_INFO *ngram);
boolean init_ngram_bin(NGRAM_INFO *ndata, char *ngram_file);
boolean init_ngram_arpa(NGRAM_INFO *ndata, char *ngram_file, int dir);
boolean init_ngram_arpa_additional(NGRAM_INFO *ndata, char *bigram_file);
void set_unknown_id(NGRAM_INFO *ndata, char *str);

void print_ngram_info(FILE *fp, NGRAM_INFO *ndata);

#include <sent/vocabulary.h>
boolean make_voca_ref(NGRAM_INFO *ndata, WORD_INFO *winfo);
void fix_uniprob_srilm(NGRAM_INFO *ndata, WORD_INFO *winfo);

#ifdef __cplusplus
}
#endif

#endif /* __SENT_NGRAM2_H__ */