/* A program to create form and tag dictionaries from tagged format.
   File-file type input: i.e. take input from a set of files whose
   names are specified in a file called 'filelist.tag'.  Entries are allocated
   to members of a set of linked lists headed by pointers arranged in a
   2D array.  Assignment is according to first and second characters of
   keystrings; mode value controls whether tag or form is treated as the
   keystring.  Correct position for a new entry in a list is
   established; if a new tag and / or form, then the new entry is inserted
   into the last at the correct position.

   This version offers the option of basic statistical information in
   the output. This option prints two numbers after each form: the
   first is the number of tokens of the form, the second is the number of
   texts in which the form is attested.

   An option to print line-breaks in the entry lists is also provided.
   
   IHD/KW Version 3.1 -- December 1997 

   = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =

   An option allows material within ( ) in input - e.g. elaborated
   tagging - to be filtered from the output. Calling 'Dicmaker3p2 1',
   where the parameter value is 1 causes material in tags not to be printed
   in the output file. 'Dicmaker3p2 0' gives full tags in the output,
   i.e. material in ( ) is included in the output tags.
   
   IHD/KW Version 3.2 -- March 2005

   = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =

   An option is included to print tags or forms in a format suitable for
   loading into a MySQL table. The field separator is '|' and field
   content is enclosde by '"'. The options are invoked by using 100 as
   the mode value for selecting tags and 200 for selecting forms.

   IHD/KW Version 3.2.1 -- April 2006

   */


#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <string.h>

#define STRLEN 40
#define BATCHES 127
#define MAXLEN 65

/* input is file-from-file type: i.e. take input from a
   set of files whose names are specified in a file called 'filelist.tag' */

FILE *dbf, *inf, *outf, *flist;

struct entry{
  char *keystr;
  char *substr;
  int token_tal;
  int text_tal;
  int textinc;
  int text_number;
  struct entry *s;
};

int select_mode();
void do_fail( char *);
void reset_textinc();
void read_dictionary();
void fill_list();
char *read_string( int);
int numlen( int);
void split_string( char *);
char *filter_string( char *);
void print_dictionary();
void make_MySQL_input();
void print_textnumbers();

struct entry *p[BATCHES][BATCHES];

int at_head, docno, key, mode;
int phase, print_code, qv=0, skip_elaborations, text_count=0;
char *dataname;

main( int argc, char *argv[])
{

int i, j, k;

  if( (flist = fopen("filelist.tag","r"))==0)
    do_fail( "filelist.tag");

  skip_elaborations = atoi( argv[1]);
  
  mode = select_mode();

  if( mode < 100)
  {
    printf("\n\n\t][ Do you wish entry lines in output to contain breaks?");
    printf("\n\n\t][ NO = 0; YES = 1");
    printf("\n\n\t >> ");

    scanf("%d",&print_code);
  }
  else print_code = 0;

  phase = 0;

  printf("\n\n\n\tD o I n G . . . \n\n");

  for(i=0; i<BATCHES; ++i)  /* initialize alphabetic lists */
  {
    for( j=0; j<BATCHES; ++j)
         p[i][j] = NULL;
  }

  phase++;

  if( (mode>1 && mode<4) || (mode>11 && mode<14))
    read_dictionary();

  phase++;

  dataname = (char *)malloc(STRLEN);
 
  fscanf(flist,"%s",dataname);

  k = 0;

  while( *dataname!='!')
  {
    if( (inf = fopen(dataname,"r"))==0)
      do_fail( dataname);


/*  printf("\n\t][  Reading from '%s'\n\n",dataname);*/

    if( !feof(flist) && !feof(inf))
      fill_list();

    fclose(inf);

    fscanf(flist,"%s",dataname);

  }

  fclose(flist);

  if( mode < 100)
  {
    if( (outf = fopen("basedic","w"))==0)
      do_fail( "basedic");

    if( mode==4 || mode==5)
      print_textnumbers();
    else
      print_dictionary();

    fclose( outf);
  }
  else
  {
    if( (dbf = fopen("basedic.mysql","w"))==0)
      do_fail( "basedic.mysql");

    make_MySQL_input();

    fclose( dbf);
  }

  printf("\n\n\tD o N e\n\n");

  return(0);

} /* end of main */

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

int select_mode()
{
int m;

  printf("\n\nSelect MODE ...\n");
  printf("\n");
  printf("\n\t][  Make\tDictionary by TAG 	= 0\n");
  printf("\n\t][  Make\tDictionary by FORM 	= 1\n");
  printf("\n\t][  Update\tDictionary by TAG	= 2\n");
  printf("\n\t][  Update\tDictionary by FORM	= 3\n");
  printf("\n\t][  Include text numbers\tby TAG	= 4\n");
  printf("\n\t][  Include text numbers\tby FORM	= 5\n");
  printf("\n\t][  Make\tFile for MySQL by TAG	= 100\n");
  printf("\n\t][  Make\tFile for MySQL by FORM	= 101\n");
  printf("\n\n\t<> To obtain basic statistical data in the output,");
  printf("\n\t<> to the mode values 0 and 1 ONLY, add 10.");

  printf("\n\n\t][  MODE >> ");

  scanf("%d",&m);

  if(m<0 || (m>5 && m<10) || (m>11 && (m<100 || m>101)))
  {
    printf("\n\n\t][  Mode value invalid.  Run terminated.\n");
    exit(0);
  }

  if( m > 1)
    key = m%2;
  else
    key = m;

/*MSG*/
printf("\n\n\t][  MODE = %d  KEY = %d\n\n",m,key);

  return(m);

} /* end of select_mode */

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */


void do_fail( fname)

char *fname;

{
  printf( "\n\n\t\t\t* * * * * FAILURE OPENING");
  printf( "\n\t\t%s\n",fname);
  printf( "\n\n\t\t\t* * * * *\n\n");

  exit( 0);
}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void reset_textinc()

{
struct entry *mp;
int i, j;

  for( i=0; i<BATCHES; ++i)
    for(j=0; j<BATCHES; ++j)
    {
      mp = p[i][j];

      while( mp)
      {
        mp->textinc = 0;
        mp = mp->s;
      }
    }
}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void read_dictionary()

{
struct entry *u, *q[BATCHES][BATCHES];
int d, v, w, x;
char *head_str, *k;


    if( !key)
    {

/*MSG*/
printf("\n\n\t][  STARTING TO READ TAGDIC . . .\n\n");

      if( (inf = fopen("tagdic","r"))==0)
        do_fail( "tagdic");
    }
    else
    {

/*MSG*/
printf("\n\n\t][  STARTING TO READ DICF . . .\n\n");

      if( (inf = fopen("dicf","r"))==0)
        do_fail( "dicf");
    }

    while( !feof(inf))
    {
      head_str = read_string( 0);

      k = head_str;      /* find batch cells */
      v = *k;
      *k++;
      w = *k;

      while( !feof(inf) && at_head==0)
      {
        u = (struct entry *)malloc(sizeof(struct entry));

        u->keystr = (char *)malloc(STRLEN);
        strcpy(u->keystr,head_str);
        u->substr = read_string( 0);

        if( !p[v][w])
        {
          u->s = p[v][w];
          p[v][w] = u;
          q[v][w] = p[v][w];
        }
        else
        {
          u->s = q[v][w]->s;
          q[v][w]->s = u;
          q[v][w] = q[v][w]->s;
        }
      }
    }

    fclose( inf);

/*MSG*/
if( !key)
  printf("\n\n\t][  TAGDIC READ IN.");
else
  printf("\n\n\t][  DICF READ IN.\n\n");
  
}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void fill_list()

{
struct entry *rp, *tp, *z;
int a, b, x, y;
char *k;

  while( !feof(inf))
  {
    z = (struct entry *)malloc(sizeof(struct entry));

    if(!z)    
    {
      printf("\n\t][  Out of space ... RUN STOPPED.");
      exit(0);
    }

    if( key==0)
    {
      z->keystr = read_string( skip_elaborations);
      z->substr = read_string( 0);
    }
    else
    {
      z->substr = read_string( skip_elaborations);
      z->keystr = read_string( 0);
    }

    z->text_number = docno;

/*if( z)
  printf( "\n%s %s",z->keystr,z->substr);
*/
    if( qv<0 || qv>1)
    {
      printf("\t][  FLAG ERROR!\n\n");
      printf("\t][  Last strings read: %s %s\n\n",z->keystr,z->substr);
      printf("\t][  Run terminated\n");
      printf("\t][  Current filename is '%s'\n\n",dataname);
      exit(0);
    }

    k = z->keystr;
    x = *k;
    *k++;
    y = *k;

    if( mode>9)
    {
      z->token_tal = 0;
      z->text_tal = 0;
    }

    if( !p[x][y])
    {
      if( mode>9)
      {
        z->token_tal++;
        z->text_tal++;
        z->textinc = 1;
      }
      z->s = p[x][y];
      p[x][y] = z;
    }
    else
    {

      /* now find position in list for new entry */

      rp = tp = p[x][y];

      while( tp && ((a=strcmp(z->keystr,tp->keystr)) > 0))
      {
        rp = tp;
        tp = tp->s;
      }

      if( a==0)
        while( tp && ((a=strcmp(z->keystr,tp->keystr))==0) && ((b=strcmp(z->substr,tp->substr)) > 0))
        {
          rp = tp;
          tp = tp->s;
        }

      if( mode>9)
        if( tp && (a==0 && b==0))
        {
          tp->token_tal++;

          if( tp->textinc==0)
          {
            tp->text_tal++;
            tp->textinc = 1;
          }
        }

      if( !(a==0 && b==0))
      {
        if( mode>9)
        {
          z->token_tal++;
          z->text_tal++;
          z->textinc = 1;
        }

        z->s = tp;          /* ... link in new entry to list */

        if( p[x][y]==tp)
          p[x][y] = z;
        else
          rp->s = z;
      }

    } /* end if */
  }

} /* end of fill_list */

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

char *read_string( filter)

int filter;
{
char *ct, *t;
int c;

    while( ((c=fgetc(inf)) != EOF) && !(c=='#' || c=='$' || c=='_' || c==' '))
      ;

    if(c==' ')
    {

      while( isspace( c=fgetc(inf)))
       ;

      if( c != '$')
      {
        if(c=='\?' || c=='\'' || c=='!' || c==';' || c=='{')
          while( ((c=fgetc(inf)) != EOF) && !(c=='$'))
           ;
      }

      if( !feof(inf) && c != '$')
        ungetc(c,inf);
  
    }

    if(c=='#')
    {
      if( mode>9)
      {
        text_count++;
        reset_textinc();
      }

      while( !isdigit(c))
        c=fgetc(inf);

      ungetc(c,inf);
      fscanf( inf,"%d",&docno);

/*    printf("\n\t\t][  text = %d\n",docno);*/

      while( ((c=fgetc(inf)) != EOF) && !(c=='$'))
        ;
    }

    if( !feof(inf) && (phase == 2))
    {
      if(c=='$') qv++;
      if(c=='_') qv--;
    }

    at_head = 0;

    t = (char *)malloc(STRLEN);

    if(t != 0)
    {
      ct = t;

      while( ((c = fgetc(inf)) != EOF) && !(c == '\n' || c == ' ' || c == '_'))
      {
        if( c=='(' && filter==1)
        {
/*        if( !feof(inf))
            ungetc( c,inf);
*/
          while( ((c=fgetc(inf))!=EOF) && !(c==')' || c == '\n' || c == ' ' || c == '_'))
            ;
        }

        if( c==')' && filter==1)
          continue;

        if( c=='*' || c=='\\')
          continue;

        if( ct-t < STRLEN-1)
          *ct++ = c;
      }

      switch(c)
      {
        case '\n': while( ((c=fgetc(inf)) != EOF) && !(c=='$' || c=='#'))
                                            ;
                   at_head = 1;

                   if( !feof(inf))
                     ungetc(c,inf);
                   break;

        case '_' :

        case ' ' : ungetc(c,inf);
                   break;
        default  : ;
      }

      *ct = 0;
      if(c==EOF && ct==t)
        return(0);
    }
    return(t);
}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

int numlen( t)

int t;

{
int s, r;

  r = 1;
  s = 0;

  while( r<t)
  {
    r = r*10;
    ++s;
  }

  return( s);

}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void print_dictionary()

{
int i, j, line_len, q;
struct entry *lp, *mp;

/*MSG*/

  printf("\n\n\t][  PRINTING");
  if( key==0)
    printf(" TAG");
  else
    printf(" FORM");

  printf(" DICTIONARY");

  if( mode>9)
    printf(" WITH STATISTICAL INFORMATION");

  printf("\n\n");

  for(i=0; i<BATCHES; ++i)
  {
    for( j=0; j<BATCHES; ++j)
    {
      lp = mp = p[i][j];

      while(mp)
      {
        line_len = 0;

        fprintf(outf,"$%s %s",mp->keystr,mp->substr);

        if( print_code==1)
          line_len = line_len + strlen( mp->keystr) + strlen(mp->substr) + 2;

        if( mode>9)
        {
          fprintf(outf," %d %d",mp->token_tal,mp->text_tal);
          if( print_code==1)
            line_len = line_len + numlen( mp->token_tal) + numlen( mp->text_tal) + 2;
        }

        mp = mp->s;

        while(mp && (strcmp(lp->keystr,mp->keystr)==0))
        {
          fprintf(outf," %s",mp->substr);

          if( print_code==1)
            line_len = line_len + 1 + strlen(mp->substr);

          if( mode>9)
          {
            fprintf(outf," %d %d",mp->token_tal,mp->text_tal);
            if( print_code==1)
              line_len = line_len + numlen(mp->token_tal) + numlen(mp->text_tal) + 2;
          }

          mp = mp->s;

          if( print_code==1 && mp && strcmp(lp->keystr,mp->keystr)==0 && line_len > MAXLEN)
          {
            fprintf( outf,"\n ");
            line_len = 1;
          }
        }

        lp = mp;

        fprintf(outf,"\n");
      }
    }
  }

} /* end of print_dictionary */

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void split_string( kstring)

char *kstring;

{
  char *a, *g, *gram, *l, *lex;

  gram = (char *)malloc( STRLEN);
  lex = (char *)malloc( STRLEN);

  g = gram;
  l = lex;
  a = kstring;

  while( *a && (*a != '/'))
    *l++ = *a++;

  *l = 0;

  while( *a)
    *g++ = *a++;

  *g = 0;

  fprintf( dbf,"|");

  if( *lex)
    fprintf( dbf,"\"%s\"|", lex);

  if( *gram)
    fprintf( dbf,"\"%s\"|", gram);

  fprintf( dbf,"||\n");

}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

char* filter_string( kstring)

char *kstring;
{
char *a, *b, *kleanstr;

  kleanstr = (char *)malloc(STRLEN);
  b = kleanstr;
  a = kstring;

  if( *a=='+')
    *b++ = *a++;

/*%%%*/

  while( *a)
  {
    if( *a=='+')
    {
      *a++;

/*      if( *a!=0)
      {
        *b++ = *a++;
      }
      else*/

      if( *a==0)
        *a--;
        *b++ = *a++;
    }
    else
    if( *a=='\"' || *a=='\\')
    {
      *b++ = '\\';
      *b++ = *a++;
    }
    else
    if( *a)
      *b++ = *a++;
  }

  *b = 0;

  return( kleanstr);
}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void make_MySQL_input()

{
int i, j;
struct entry *lp, *mp;
char *cleanstr;

  cleanstr = (char *)malloc( STRLEN);

  for(i=0; i<BATCHES; ++i)
  {
    for( j=0; j<BATCHES; ++j)
    {
      lp = mp = p[i][j];

      while( mp)
      {

        if( mode==100)
        {
          split_string( mp->keystr);
        }
        else
        {
          cleanstr = filter_string( mp->keystr);
          fprintf( dbf,"|\"%s\"|||\n",cleanstr);
        }

        mp = mp->s;

        while( mp && (strcmp( lp->keystr,mp->keystr)==0))
          mp = mp->s;
        
        lp = mp;

      }
    }
  }

}

/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = */

void print_textnumbers()

{
int i, j, line_len, q;
struct entry *lp, *mp;
char *cleanstr;

  for(i=0; i<BATCHES; ++i)
  {
    for( j=0; j<BATCHES; ++j)
    {
      lp = mp = p[i][j];

      while(mp)
      {
        line_len = 0;

        if( mode==4)
        {
          cleanstr = filter_string( mp->substr);
          fprintf(outf,"$%s : %s ; %d",mp->keystr,cleanstr,mp->text_number);
        }
        else
        {
          cleanstr = filter_string( mp->keystr);
          fprintf(outf,"$%s : %s ; %d",cleanstr,mp->substr,mp->text_number);
        }

        if( print_code==1)
          line_len = line_len + strlen( mp->keystr) + strlen(mp->substr) + 6;

        mp = mp->s;

        while(mp && (strcmp(lp->keystr,mp->keystr)==0))
        {
          if( mode==4)
          {
            cleanstr = filter_string( mp->substr);
            fprintf(outf," %s ; %d",cleanstr,mp->text_number);
          }
          else
            fprintf(outf," %s ; %d",mp->substr,mp->text_number);

          if( print_code==1)
            line_len = line_len + 1 + strlen(mp->substr) + 4;

          mp = mp->s;

          if( print_code==1 && mp && strcmp(lp->keystr,mp->keystr)==0 && line_len > MAXLEN)
          {
            fprintf( outf,"\n ");
            line_len = 1;
          }
        }

        lp = mp;

        fprintf(outf,"\n");
      }
    }
  }

} /* end of print_dictionary */

/* ( ( ( ( ( ( ( ( ( ( E N D * OF * P R O G R A M ) ) ) ) ) ) ) ) ) ) */