source: src/tok.c/tok.c @ 5f4d9c3

#include <cctype>
#include <cstdio>
#include <cstring>
#include <clocale>
#include <cstdlib>
//#include "../lib/iotools.h"

#include "cmdline.h"


char buf[257];
int filepos=0;
char* tokstart;
char* tokend;
char tmp;
char tag;


gengetopt_args_info args;


inline
void printtoken(char tag)
{
  tmp=*tokend;
  *tokend='\0';
  printf("%04d %02d %c %s\n", filepos, tokend-tokstart, tag, tokstart);
  *tokend=tmp;
  filepos+=tokend-tokstart;
  tokstart=tokend;
  if(args.interactive_flag) fflush(stdout);
}

main(int argc, char** argv)
{

  if (cmdline_parser(argc, argv, &args) != 0)
    exit(1);

  printf("inter:%d\n",args.interactive_flag);

  //  process_common_options(&args, argv[0]);
  //  process_tok_options(args);

  setlocale(LC_CTYPE,"");
  setlocale(LC_COLLATE,"");

  while(fgets(buf,256,stdin))
  {

    tokstart=tokend=buf;
    while(*tokend)
    {
      char *prev=tokend;
      ++tokend;
      if(isalpha(*prev) && !isalpha(*tokend))
         printtoken('W');
      else if(isdigit(*prev) && !isdigit(*tokend))
        printtoken('N');
      else if(isspace(*prev))
        {
          switch(*prev)
            {
            case ' ': *prev='_'; break;
            case '\t':*prev='t'; break;
            case '\r':*prev='r'; break;
            case '\f':*prev='f'; break;
            case '\n':*prev='n';
            }
          if(!isspace(*tokend))
            printtoken('S');
        }
      else if(ispunct(*prev))
         printtoken('P');
    }
  }

  cmdline_parser_free(&args);
}