misclib_testcode::tokenizertest Class Reference

Test the tokenizer and related stuff. More...

#include <tokenizertest.h>

Collaboration diagram for misclib_testcode::tokenizertest:

List of all members.

Public Member Functions
void	test00 ()
	Given a paragraph as a string parse into lines.
void	test01 ()
	Use STL to erase a token in a string.
void	test02 ()
	Use STL to insert white space around a token in a string.
void	test03 ()
	Demonstrate the atomize function.
void	test04 ()
	Demonstrate the stripcomment function.
void	test05 ()
void	test06 ()
void	test07 ()
	Testing the trim function.
void	test08 ()
	Testing the tokenizer on a real VRML file.
void	test09 ()
	Test comparewithoutspace(s1,s2).
Static Public Member Functions
static void	test10 ()
	atomize_next(<mytag>) to find tagged data.
static int	unittest01 ()
	Exploring reading and writing tagged field with tokenizer::atomize_next_tag().
static int	unittest02 ()
	tokenizerlocal: reading and writing a tagged string.
static int	unittest03 ()
	Reading/writing multiple <obj>.
static int	unittest04 ()
Static Public Attributes
static string	doc []
	Brief description of each test.

---

Detailed Description

Test the tokenizer and related stuff.

Definition at line 10 of file tokenizertest.h.

---

Member Function Documentation

void tokenizertest::test00

(

)

Given a paragraph as a string parse into lines.

Definition at line 37 of file tokenizertest.cpp.

References tokenizer::seq, and tokenizer::subtract().

{
  tokenizer ss;
  string tmp("This is a silly\n sentence that is accross \n several lines.");
  tmp += " Will the parser split this string?\n It should unless I screwed up.";

  ss.seq.push_back(tmp);
  ss.subtract("\n");
  cout << ss << endl;
}

void tokenizertest::test01

(

)

Use STL to erase a token in a string.

Definition at line 48 of file tokenizertest.cpp.

References SHOW.

{
  cout << "Erase token irrespective of white space." << endl;

  string s1="brownThe quick brown fox";

  string s2="brown";

  cout << SHOW(s1) << endl;
  cout << SHOW(s2) << endl;

  string::size_type i = 0;

  i = s1.find(s2,i);
  while (i!=string::npos)
  {
    s1.erase(i, sizeof(s2)+1);
    i = s1.find(s2,i);
  }
  
  cout << SHOW(s1) << endl;
}

void tokenizertest::test02

(

)

Use STL to insert white space around a token in a string.

Definition at line 71 of file tokenizertest.cpp.

References SHOW.

{
  cout << "Find token and insert white space around it." << endl;

  string s1="brownThe quick brown fox";

  string s2="brown";

  cout << SHOW(s1) << endl;
  cout << SHOW(s2) << endl;

  string::size_type i = 0;

  i = s1.find(s2,i);
  while (i!=string::npos)
  {
    s1.insert(i+sizeof(s2)+1," ");
    s1.insert(i," ");
    ++i; ++i;
    i = s1.find(s2,i);
  }
  
  cout << SHOW(s1) << endl;
}

void tokenizertest::test03

(

)

Demonstrate the atomize function.

Definition at line 96 of file tokenizertest.cpp.

References tokenizer::atomize(), and tokenizer::seq.

{
  cout << "Testing tokenizer::atomize function." << endl;
  tokenizer ss;
  ss.seq.push_back("(a+b)*d+e");
  ss.seq.push_back("+gh");
  cout << "Print the current state" << endl;
  cout << ss << endl << endl;
  cout << "Atomize on +" << endl;
  ss.atomize("+");
  cout << ss << endl;
}

void tokenizertest::test04

(

)

Demonstrate the stripcomment function.

Definition at line 109 of file tokenizertest.cpp.

References tokenizer::printdelimiter, tokenizer::remove(), tokenizer::seq, tokenizer::stripcomment(), and tokenizer::subtract().

{
  cout << "Testing tokenizer::stripcomment function." << endl;
  tokenizer ss;
  ss.printdelimiter="#\n";
  ss.seq.push_back("// A silly class.");
  ss.seq.push_back("class fred  // A comment");
  ss.seq.push_back("{ public:");
  ss.seq.push_back("  fred();  //Constructor");
  ss.seq.push_back("  ff() { z*z; } // another one.");
  ss.seq.push_back("}");
  cout << ss << endl;
  cout << endl;
  cout << "Striping the comments" << endl;

  ss.stripcomment("//");

  cout << ss << endl << endl;
  cout << "Remove line" << endl;

  ss.remove("");

  //ss.remove_if();

  cout << ss << endl << endl;

  cout << "Subtract space" << endl;
  ss.subtract(" ");
  cout << ss << endl;
}

void tokenizertest::test05

(

)

Definition at line 140 of file tokenizertest.cpp.

References tokenizer::readaslines(), and tokenizermisc::readfile().

{
  string fname("crp.txt");
  string s;
  if (!tokenizermisc::readfile(s,fname))
  {
    cout << "error: Can not open " << fname << "." << endl;
    return;
  }

  tokenizer ss;

  ss.readaslines(s);
  cout << ss << endl;

/*
  string fname("crp.txt");
  ifstream targ(fname.c_str());

  if (!targ)
  {
    cout << "error: Can not open " << fname << "." << endl;
    return;
  }

  string s;
  char c;
  while (targ.get(c))
    s.push_back(c);

  cout << "Printing the file" << endl;
  cout << s << endl;
*/
}

void tokenizertest::test06

(

)

Definition at line 191 of file tokenizertest.cpp.

References tokenizer::atomize(), tokenizer::printdelimiter, tokenizer::readaslines(), tokenizermisc::readfile(), tokenizer::remove_if(), tokenizer::stripcomment(), and tokenizer::subtract().

{
  string filename("head.wrl");
  string s;
  if (!tokenizermisc::readfile(s,filename))
  {
    cout << "error: Can not open " << filename << "." << endl;
    return;
  }

  cout << "File read" << endl;

  tokenizer tokenstream;
  tokenstream.readaslines(s);
  //cout << tokenstream << endl;

  tokenstream.stripcomment("#");

  tokenstream.subtract(",");
  tokenstream.atomize("{");
  tokenstream.atomize("}");
  tokenstream.atomize("[");
  tokenstream.atomize("]");

  //tokenstream.seq.remove("\n");
  tokenstream.printdelimiter="@\n";

  tokenstream.remove_if(spacerdelete<>());
  //tokenstream.seq.remove_if(spacerdelete<>());

/*
  liststringi k = tokenstream.seq.begin();
  liststringi kend=tokenstream.seq.end();
  for ( ; k!=kend; ++k)
  {
    if
*/
  
cout << "***" << endl;
cout << tokenstream << endl;

}

void tokenizertest::test07

(

)

Testing the trim function.

Definition at line 234 of file tokenizertest.cpp.

References tokenizer::printdelimiter, tokenizer::seq, and tokenizer::trim().

{
  cout << "Testing spacetrim<>" << endl;
  tokenizer ss;
  ss.printdelimiter="@\n";
  ss.seq.push_back("Nothing");
  ss.seq.push_back(" fku");
  ss.seq.push_back("  This is a sentence. ");
  ss.seq.push_back(" a");
  ss.seq.push_back("a ");
  ss.seq.push_back(" a ");
  ss.seq.push_back(" ");
  ss.seq.push_back("      ");

  ss.trim();

  cout << ss << "@" << endl;
}

void tokenizertest::test08

(

)

Testing the tokenizer on a real VRML file.

Definition at line 253 of file tokenizertest.cpp.

References tokenizer::atomize(), dbg, tokenizer::printdelimiter, tokenizer::readaslines(), tokenizermisc::readfile(), tokenizer::remove_if(), tokenizer::stripcomment(), tokenizer::subtract(), and tokenizer::trim().

{
  string filename("head2.wrl");
  string s;
  if (!tokenizermisc::readfile(s,filename))
  {
    cout << "error: Can not open " << filename << "." << endl;
    return;
  }

  tokenizer tokenstream;
  tokenstream.readaslines(s);
  //cout << tokenstream << endl;

  tokenstream.stripcomment("#");

  tokenstream.subtract(",");
  tokenstream.atomize("{");
  tokenstream.atomize("}");
  tokenstream.atomize("[");
  tokenstream.atomize("]");

  //tokenstream.seq.remove("\n");
  tokenstream.printdelimiter="@\n";

  tokenstream.remove_if(spacerdelete<>());

  tokenstream.trim();

  tokenstream.subtract(" ");

  messagefile dbg("debug.txt",true);
  dbg() << tokenstream << endl;
}

void tokenizertest::test09

(

)

Test comparewithoutspace(s1,s2).

Definition at line 288 of file tokenizertest.cpp.

References tokenizer::atomize(), tokenizermisc::comparewithoutspace(), tokenizer::remove_if(), tokenizer::reset(), SHOW, tokenizer::trim(), and tokenizer::trim_and_prune().

{
  string s1 = "The quick brown fox jumped over the lazy dog.\n";
  string s2 = "   The quick brown fox\njumped     over the lazy dog.";

  tokenizer t1(s1);
  tokenizer t2(s2);

  t1.atomize(" ");
  t1.atomize("\n");
  t1.atomize("\t");
  t1.trim_and_prune();
  //t1.trim();
  //t1.remove_if(spacerdelete<>());
  t1.reset();
  for ( ; !t1; ++t1 )
    cout << "*" << t1() << "*" << endl;
  cout << endl << endl;

  t2.atomize(" ");
  t2.atomize("\n");
  t2.atomize("\t");
  t2.trim();
  t2.remove_if(spacerdelete<>());
  t2.reset();
  for ( ; !t2; ++t2 )
    cout << "*" << t2() << "*" << endl;
  cout << endl << endl;

  cout << t1 << "@" << endl;
  cout << t2 << "@" << endl;

  cout << SHOW(t1==t2) << endl;

  cout << SHOW(tokenizermisc::comparewithoutspace(s1,s2)) << endl;

}

void tokenizertest::test10

(

)

[static]

atomize_next(<mytag>) to find tagged data.

Definition at line 391 of file tokenizertest.cpp.

References tokenizer::atomize_next(), stl_iterator< Cont, Tp >::reset(), tokenizer::reset(), and tokenizer::seq.

{
  string s1;
  example01(s1);
  tokenizer tk(s1);

  cout << "Display string:" << endl;
  cout << tk << endl;
  cout << endl;
  cout << "atomize_next <startcolor>:" << endl;
  tk.reset();
  tk.atomize_next("<startcolor>");

  //cout << SHOW(tk()) << endl;

  liststringiterator i1(tk.seq);
  for (i1.reset(); !i1; ++i1)
    { cout << "*" << *i1 << "*" << endl; }

  cout << "atomize_next </startcolor>:" << endl;
  tk.atomize_next("</startcolor>");

  for (i1.reset(); !i1; ++i1)
    { cout << "*" << *i1 << "*" << endl; }
}

int tokenizertest::unittest01

(

)

[static]

Exploring reading and writing tagged field with tokenizer::atomize_next_tag().

Definition at line 417 of file tokenizertest.cpp.

References assertreturnOS, tokenizer::atomize(), tokenizer::atomize_next_tag(), tokenizer::current, endln, tokenizer::find(), print(), tokenizer::reset(), tokenizerlocal::scope(), tokenizer::seq, SHOW, and tokenizer::trim_and_prune().

{
  {
    cout << "Test 1.1" << endln;
    string s1;
    example01(s1);
    tokenizer tk(s1);

    liststringi i1;
    liststringi i2;
    tk.reset();
    bool res;
    res=tk.atomize_next_tag(i1,i2,"startcolor");
    cout << "Atomize <startcolor> and </startcolor>" << endl;
    cout << SHOW(res) << endl;
    if (res==false)
      return 1;

    cout << "Read the tag." << endl;
    cout << "  check 3 strings." << endl;
    liststringi i3(i1);
    ++i3; ++i3;
    if (i3!=i2)
      return 1;
    --i3;
    cout << "tag data=\"" << *i3 << "\"" << endl;
    cout << "Writing a value back" << endl;
    *i3 = "0.1 0.1 0.1";

    tk.trim_and_prune();

    string s2 = (stringc)tk;
    cout << tk << endl;
  }

  {
    cout << "Test 1.2" << endln;
    string s1;
    example01(s1);
    tokenizer tk(s1);

    tk.reset();
    tk.atomize("<settings>");
    tk.atomize("<pipes>");

    cout << print(tk.seq,"*") << endln;
    string::size_type k1;

    tk.reset();
    assertreturnOS( tk.find(k1,"matrixD2") );
    cout << SHOW( *tk.current ) << " " << SHOW(k1) << " ";
    cout << SHOW(tk.current->substr(k1,8)) << endl;
    assertreturnOS(tk.current->substr(k1,8)=="matrixD2");
    assertreturnOS( tk.find(k1,"<n>",k1) );
    cout << SHOW( *tk.current ) << " " << SHOW(k1) << " ";
    cout << SHOW(tk.current->substr(k1,3)) << endl;
    assertreturnOS(tk.current->substr(k1,3)=="<n>");

    assertreturnOS( tk.find(k1,"<pipes>",k1) );
    cout << SHOW( *tk.current ) << " " << SHOW(k1) << endln;
  }

  {
    cout << "Test 1.3" << endln;
    string s1;
    example01(s1);
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();
    assertreturnOS( tl.scope("settings") );
    assertreturnOS( tl.scope("pipes") );

    tokenizerfind tf(tk,"<");
    tf.reset();
    for ( ; !tf; ++tf)
      cout << SHOW(tf()) << " " << SHOW(tf.index) << endln;

    cout << "find(\"matrixD2\",\"n\")" << endl;
    
    tk.reset();
    assertreturnOS( tf.find("<matrixD2>","<n>") );
    cout << SHOW(tf()) << " " << SHOW(tf.index) << endln;
    cout << SHOW( tf().substr(tf.index,3) ) << endl;

  }

  return 0;
}

int tokenizertest::unittest02

(

)

[static]

tokenizerlocal: reading and writing a tagged string.

Definition at line 507 of file tokenizertest.cpp.

References assertreturnOS, endln, tokenizerlocal::erasetag(), tokenizerlocal::i1, tokenizerlocal::i2, tokenizerlocal::read(), tokenizerlocal::ref, tokenizerlocal::reset(), tokenizer::reset(), tokenizerlocal::scope(), tokenizerlocal::scopesearch(), tokenizer::seq, SHOW, and tokenizerlocal::writetag().

{
  string s1;
  example01(s1);

  cout << "Testing tokenizerlocal::scope(stringc&);" << endl;
  cout << "Looking to see list of strings with *<matrixD2>* ... *</matrixD2>" << endl;

  {
    cout << "TEST 2.1" << endl;
    cout << "Atomizing tag" << endln;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();
    assertreturnOS( tl.scope("matrixD2") );

    liststringiterator i1(tk.seq);
    for (i1.reset(); !i1; ++i1)
      { cout << "*" << *i1 << "*" << endl; }
  }

  {
    cout << "TEST 2.2" << endl;
    cout << "Atomizing tag depth of 2." << endln;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset(); 
    assertreturnOS( tl.scope("matrixD2","m") );

    liststringiterator i1(tk.seq);
    for (i1.reset(); !i1; ++i1)
      { cout << "*" << *i1 << "*" << endl; }
  }

  {
    cout << "TEST 2.3" << endl;
    cout << "Reading and writing a tag at depth of 2." << endln;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset(); 

    string val;

    assertreturnOS( tl.read(val,"startcolor") );
    cout << "*" << SHOW(val) << "*" << endl;

    assertreturnOS( tl.read(val,"matrixD2","n") );
    cout << "*" << SHOW(val) << "*" << endl;

    cout << SHOW(*tl.i1) << SHOW(*tl.i2) << endl;
    cout << SHOW(*tl.ref) << endl;

    tl.reset();
    tl.writetag("i am starting to dislike working on the parser", "n");
    cout << SHOW(*tl.ref) << endl;

    cout << tk << endl;
  }

  {
    cout << "TEST 2.4" << endl;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();
    cout << "Erasing the <settings> tag." << endl;

    bool res;
    res = tl.scopesearch("settings");
    cout << SHOW(res) << endl;
    cout << "Did not expect to find settings as not atomized." << endl;
    assertreturnOS(res==false);
    tk.reset();
    res = tl.scope("settings");
    assertreturnOS(res==true);
    cout << SHOW(res) << " " << SHOW(*tl.i1) << " " << SHOW(*tl.i2) << endl;
    assertreturnOS(tl.erasetag("settings"));

    cout << tk << endl;
  }

  return 0;
}

int tokenizertest::unittest03

(

)

[static]

Reading/writing multiple <obj>.

..</obj>'s

Definition at line 594 of file tokenizertest.cpp.

References assertreturnOS, misclib_testcode::InventoryObj::cost, misclib_testcode::InventoryObj::desc, endln, misclib_testcode::InventoryObj::item, print(), misclib_testcode::InventoryObj::quantity, tokenizerlocal::read(), tokenizerlocal::ref, tokenizerlocal::reset(), tokenizer::reset(), tokenizerlocal::scope(), SHOW, misclib_testcode::InventoryObj::sn, stringfrom(), and tokenizerlocal::writetag().

{
  string s1;
  example02(s1);

  {
    cout << "TEST 3.1" << endl;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();

    cout << "Listing the employees" << endln;
    cout << "  reading multiple <emp> string </emp>" << endln;

    string name;
    assertreturnOS(tl.read(name,"field","name"));
    cout << SHOW(name) << endl;
    string ei;
    list<string> names;
    for ( ; tl.read(ei,"emp"); )
      names.push_back(ei); 
    cout << "names:" << print(names," ") << endl;
    
    cout << "Edit by reading the whole list in, editing it";
    cout << " there and then re writing it." << endl;

    names.erase(find(names.begin(),names.end(),"Samantha"));

    names.push_back("Zelda");
    names.push_back("Ba");

    cout << "###" << print(names," ") << endl;


    liststringiterator namesi(names); 

    string s2;
    s2 = ("<name>" + name + "</name>");
    for ( ; !namesi; ++namesi)
    {
 assert(!namesi);
      s2 += ("<emp>"+*namesi+"</emp>");
    }
    tk.reset();
    tl.writetag(s2,"field");

    cout << tk << endl;
  }

  {
    string s0;
    example03(s0);
    cout << "TEST 3.2" << endl;
    cout << "Reading/writing multiple <obj><Item>...</obj>'s" << endln;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);
    string s2;
    tl.read(s2,"Inventory");

    list<InventoryObj> inventory;
    InventoryObj prod;

    tokenizer tk2(s2);
    tokenizerlocal tl2(tk2);
    tokenizerlocal tl3(tk2);
    string targ;
    spacertrim<> st;
    for ( tk2.reset(); tl2.scope("obj"); ++tl2)
    {
      // Atomic read.
      if (tl3.read(prod.item,"Item")==false)
        continue;
      st(prod.item);
      if (tl3.read(targ,"Quantity")==false)
        continue;
      stringfrom(prod.quantity,targ);
      if (tl3.read(prod.desc,"Desc")==false)
        continue;
      st(prod.desc);
      if (tl3.read(prod.sn,"SN")==false)
        continue;
      st(prod.sn);
      if (tl3.read(targ,"Cost")==false)
        continue;
      stringfrom(prod.cost,targ);
      
      inventory.push_back(prod);
    } 

    list<InventoryObj>::iterator k=inventory.begin();
    for ( ; k!=inventory.end(); ++k)
      cout << k->xml() << "*" << endl;

    cout << "Making edit" << endl;
    for ( k=inventory.begin(); k!=inventory.end(); ++k)
    {
//cout << SHOW(k->item) << "*" << endl;
      if (k->item=="88329")
        k->sn = "S298RT";
    }
    
    string result;
    for ( k=inventory.begin(); k!=inventory.end(); ++k)
      { result += k->xml(); }
    
    tl.reset();
    tl.writetag(result,"Inventory");

    cout << tl.ref << endl;
  }

  return 0;
}

int tokenizertest::unittest04

(

)

[static]

Definition at line 737 of file tokenizertest.cpp.

References assertreturnOS, tokenizerlocal::debug01(), tokenizerlocal::read(), tokenizerlocal::reset(), tokenizerlocal::scope(), and SHOW3.

{
  {
    string s0;
    example03(s0);

    cout << "TEST 4.1" << endl;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);

    bool res;

    res=tl.scope("Inventory");
    assertreturnOS(res);

    string SN;
    string Desc;

    tokenizerlocal tl2(tl);
    for ( ; !tl2; ++tl2 )
    {
      res=tl2.scope("obj");
      if (res)
      {
        tokenizerlocal tl3(tl2);
        if (tl3.read(SN,"SN"))
          { cout << SHOW3(SN) << endl; }
        tl3.reset(tl2);
        if (tl3.read(Desc,"Desc"))
          { cout << SHOW3(Desc) << endl; }
        
      }
    }  
  }
  {
    string s0;
    example03(s0);

    cout << "TEST 4.2" << endl;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);

    assertreturnOS( tl.scope("Inventory") );

    string SN;
    string Desc;

    tokenizerlocal tl2(tl);
    for ( ; tl2.scope("obj"); ++tl2 )
    {
      tokenizerlocal tl3(tl2);
      if (tl3.read(SN,"SN"))
        cout << SHOW3(SN) << endl;
cout << SHOW3(tl3.debug01()) << endl;
      tl3.reset();
cout << SHOW3(tl3.debug01()) << endl;
      if (tl3.read(Desc,"Desc"))
        cout << SHOW3(Desc) << endl;
    }  
  };

  {
    string s0;
    example03(s0);

    cout << "TEST 4.3" << endl;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);

    assertreturnOS( tl.scope("Inventory") );

    string SN;
    string Desc;

    tokenizerlocalvar tl2(tl,"obj");
    for ( ; !tl2; ++tl2 )
    {
      tokenizerlocal tl3(tl2);
      if (tl3.read(SN,"SN"))
        cout << SHOW3(SN) << endl;
cout << SHOW3(tl3.debug01()) << endl;
      tl3.reset();
cout << SHOW3(tl3.debug01()) << endl;
      if (tl3.read(Desc,"Desc"))
        cout << SHOW3(Desc) << endl;
    }  
  };

  return 0;
}

---

Member Data Documentation

string tokenizertest::doc [static]

Initial value:

 
{
  "Given a paragraph as a string parse into lines.",
  "Use STL to erase a token in a string.",
  "Use STL to insert white space around a token in a string.",
  "Demonstrate the atomize function.",
  "Demonstrate the stripcomment function.",
  "",
  "",
  "Testing the trim function.",
  "Testing the tokenizer on a real VRML file.",
  "Test comparewithoutspace(s1,s2).",
  "atomize_next(<mytag>) to find tagged data.",

  "Exploring reading and writing tagged field with tokenizer::atomize_next_tag()",
  "tokenizerlocal: reading and writing a tagged string.",
  "Reading/writing multiple <obj><Item>...</obj>'s",
  "xxx",
  ""
}

Brief description of each test.

Definition at line 18 of file tokenizertest.h.

---

The documentation for this class was generated from the following files:

• misclib/tokenizertest.h
• misclib/tokenizertest.cpp