tokenizer.h

Go to the documentation of this file.

#ifndef TOKENIZER_H 
#define TOKENIZER_H

#include <cassert>
#include <string>
#include <vector>
#include <iostream>
#include <list>
#include <algorithm>
using namespace std;

#include <stringspace.h>

typedef list<string>::iterator liststringi;
typedef list<string>::const_iterator liststringic;

class tokenizermisc
{
public:

  static boolc readfile(string & str, stringc & fname);

  static boolc comparewithoutspace(stringc & s1, stringc & s2);

};


class tokenizer
{
  void atomize( liststringi & i, stringc & atom, string::size_type const k0=string::npos );
public:

  liststringi current;

  list<string> seq;

  //
  // Iterator Characteristics
  // 

  void reset();
  boolc operator ! () const; 
  string & operator * ();
  stringc & operator() () const; 
  void operator ++ ();

  //
  // Processing each element in the list. 
  //

  void atomize(stringc & atom);
  void subtract( stringc & atom );
  void tokenize();
  void stripcomment( stringc & comment );
  void remove(stringc & token);
  void remove_if();
  template< typename SPACER >
  void remove_if(SPACER spacer)
    { seq.remove_if(spacer); }

  template< typename X >
  void apply(X x)
  { 
    liststringi i = seq.begin();
    liststringic imax = seq.end();
    for ( ; i!=imax; ++i )
      { x(*i); }
  };
  
  void trim();
  void trim_and_prune();

  //
  // Miscellaneous
  //

  void extractfromcurrent
  ( 
    vector<string> & v, 
    stringc & atom 
  ) const;

  //
  // Printing
  //

  string printdelimiter;
  ostream& print(ostream& os) const;

  //
  // Reading
  //

  // Note: constructor by default calls reset().
  //   read* does not have this implemented.

  void read(stringc & data);
  void readaslines(stringc & data);
  void readaslinesgeneral(stringc & data);

  //
  // Construction
  //

  tokenizer();

  tokenizer(stringc & data);
// TODO?  tokenizer(tokenizer const & tk0);

  boolc operator == (tokenizer & t2);
  
  //
  // Parsing and tags
  //

  boolc find(string::size_type & k, stringc & atom );
 
  boolc find( string::size_type& k, stringc & atom, string::size_type const k0 );

  boolc atomize_next( stringc & atom ); 

  boolc atomize_next 
  (
    stringc & atom,
    liststringi & iend_
  );

  boolc atomize_next_tag
  (
    liststringi& i1,
    liststringi& i2,
    stringc& tag,
    liststringi& iend_
  );

  boolc atomize_next_tag
  (
    liststringi& i1,
    liststringi& i2,
    stringc& tag
  );

  /* Search for sequence bounded by two atoms. */
  boolc atomize_next
  (
    liststringi& i1,
    liststringi& i2,
    stringc& atom1,
    stringc& atom2
  );

  operator stringc ();

};

ostream & operator << (ostream & os, tokenizer const & ss);

#endif