tokenizertest.cpp

Go to the documentation of this file.

#include <iostream>
#include <fstream>
using namespace std;

#include <iteratorstypedefs.h>
#include <message.h>
#include <print.h>
#include <stringconvert.h>
#include <tokenizer.h>
#include <tokenizerfind.h>
#include <tokenizerlocal.h>
#include <tokenizertest.h>

using namespace misclib_testcode;

string tokenizertest::doc[] = 
{
  "Given a paragraph as a string parse into lines.",
  "Use STL to erase a token in a string.",
  "Use STL to insert white space around a token in a string.",
  "Demonstrate the atomize function.",
  "Demonstrate the stripcomment function.",
  "",
  "",
  "Testing the trim function.",
  "Testing the tokenizer on a real VRML file.",
  "Test comparewithoutspace(s1,s2).",
  "atomize_next(<mytag>) to find tagged data.",

  "Exploring reading and writing tagged field with tokenizer::atomize_next_tag()",
  "tokenizerlocal: reading and writing a tagged string.",
  "Reading/writing multiple <obj><Item>...</obj>'s",
  "xxx",
  ""
};

void tokenizertest::test00()
{
  tokenizer ss;
  string tmp("This is a silly\n sentence that is accross \n several lines.");
  tmp += " Will the parser split this string?\n It should unless I screwed up.";

  ss.seq.push_back(tmp);
  ss.subtract("\n");
  cout << ss << endl;
}

void tokenizertest::test01()
{
  cout << "Erase token irrespective of white space." << endl;

  string s1="brownThe quick brown fox";

  string s2="brown";

  cout << SHOW(s1) << endl;
  cout << SHOW(s2) << endl;

  string::size_type i = 0;

  i = s1.find(s2,i);
  while (i!=string::npos)
  {
    s1.erase(i, sizeof(s2)+1);
    i = s1.find(s2,i);
  }
  
  cout << SHOW(s1) << endl;
}

void tokenizertest::test02()
{
  cout << "Find token and insert white space around it." << endl;

  string s1="brownThe quick brown fox";

  string s2="brown";

  cout << SHOW(s1) << endl;
  cout << SHOW(s2) << endl;

  string::size_type i = 0;

  i = s1.find(s2,i);
  while (i!=string::npos)
  {
    s1.insert(i+sizeof(s2)+1," ");
    s1.insert(i," ");
    ++i; ++i;
    i = s1.find(s2,i);
  }
  
  cout << SHOW(s1) << endl;
}

void tokenizertest::test03()
{
  cout << "Testing tokenizer::atomize function." << endl;
  tokenizer ss;
  ss.seq.push_back("(a+b)*d+e");
  ss.seq.push_back("+gh");
  cout << "Print the current state" << endl;
  cout << ss << endl << endl;
  cout << "Atomize on +" << endl;
  ss.atomize("+");
  cout << ss << endl;
}

void tokenizertest::test04()
{
  cout << "Testing tokenizer::stripcomment function." << endl;
  tokenizer ss;
  ss.printdelimiter="#\n";
  ss.seq.push_back("// A silly class.");
  ss.seq.push_back("class fred  // A comment");
  ss.seq.push_back("{ public:");
  ss.seq.push_back("  fred();  //Constructor");
  ss.seq.push_back("  ff() { z*z; } // another one.");
  ss.seq.push_back("}");
  cout << ss << endl;
  cout << endl;
  cout << "Striping the comments" << endl;

  ss.stripcomment("//");

  cout << ss << endl << endl;
  cout << "Remove line" << endl;

  ss.remove("");

  //ss.remove_if();

  cout << ss << endl << endl;

  cout << "Subtract space" << endl;
  ss.subtract(" ");
  cout << ss << endl;
}

void tokenizertest::test05()
{
  string fname("crp.txt");
  string s;
  if (!tokenizermisc::readfile(s,fname))
  {
    cout << "error: Can not open " << fname << "." << endl;
    return;
  }

  tokenizer ss;

  ss.readaslines(s);
  cout << ss << endl;

/*
  string fname("crp.txt");
  ifstream targ(fname.c_str());

  if (!targ)
  {
    cout << "error: Can not open " << fname << "." << endl;
    return;
  }

  string s;
  char c;
  while (targ.get(c))
    s.push_back(c);

  cout << "Printing the file" << endl;
  cout << s << endl;
*/
}

/*
void messagetest::test02()
{
  string fname("crp.txt");
  string s;
  if (!readfile(s,fname))
  {
    cout << "error: Can not open " << fname << "." << endl;
    return;
  }

  cout << "Printing the file" << endl;
  cout << s << endl;
}
*/

void tokenizertest::test06()
{
  string filename("head.wrl");
  string s;
  if (!tokenizermisc::readfile(s,filename))
  {
    cout << "error: Can not open " << filename << "." << endl;
    return;
  }

  cout << "File read" << endl;

  tokenizer tokenstream;
  tokenstream.readaslines(s);
  //cout << tokenstream << endl;

  tokenstream.stripcomment("#");

  tokenstream.subtract(",");
  tokenstream.atomize("{");
  tokenstream.atomize("}");
  tokenstream.atomize("[");
  tokenstream.atomize("]");

  //tokenstream.seq.remove("\n");
  tokenstream.printdelimiter="@\n";

  tokenstream.remove_if(spacerdelete<>());
  //tokenstream.seq.remove_if(spacerdelete<>());

/*
  liststringi k = tokenstream.seq.begin();
  liststringi kend=tokenstream.seq.end();
  for ( ; k!=kend; ++k)
  {
    if
*/
  
cout << "***" << endl;
cout << tokenstream << endl;

}

void tokenizertest::test07()
{
  cout << "Testing spacetrim<>" << endl;
  tokenizer ss;
  ss.printdelimiter="@\n";
  ss.seq.push_back("Nothing");
  ss.seq.push_back(" fku");
  ss.seq.push_back("  This is a sentence. ");
  ss.seq.push_back(" a");
  ss.seq.push_back("a ");
  ss.seq.push_back(" a ");
  ss.seq.push_back(" ");
  ss.seq.push_back("      ");

  ss.trim();

  cout << ss << "@" << endl;
}

void tokenizertest::test08()
{
  string filename("head2.wrl");
  string s;
  if (!tokenizermisc::readfile(s,filename))
  {
    cout << "error: Can not open " << filename << "." << endl;
    return;
  }

  tokenizer tokenstream;
  tokenstream.readaslines(s);
  //cout << tokenstream << endl;

  tokenstream.stripcomment("#");

  tokenstream.subtract(",");
  tokenstream.atomize("{");
  tokenstream.atomize("}");
  tokenstream.atomize("[");
  tokenstream.atomize("]");

  //tokenstream.seq.remove("\n");
  tokenstream.printdelimiter="@\n";

  tokenstream.remove_if(spacerdelete<>());

  tokenstream.trim();

  tokenstream.subtract(" ");

  messagefile dbg("debug.txt",true);
  dbg() << tokenstream << endl;
}

void tokenizertest::test09()
{
  string s1 = "The quick brown fox jumped over the lazy dog.\n";
  string s2 = "   The quick brown fox\njumped     over the lazy dog.";

  tokenizer t1(s1);
  tokenizer t2(s2);

  t1.atomize(" ");
  t1.atomize("\n");
  t1.atomize("\t");
  t1.trim_and_prune();
  //t1.trim();
  //t1.remove_if(spacerdelete<>());
  t1.reset();
  for ( ; !t1; ++t1 )
    cout << "*" << t1() << "*" << endl;
  cout << endl << endl;

  t2.atomize(" ");
  t2.atomize("\n");
  t2.atomize("\t");
  t2.trim();
  t2.remove_if(spacerdelete<>());
  t2.reset();
  for ( ; !t2; ++t2 )
    cout << "*" << t2() << "*" << endl;
  cout << endl << endl;

  cout << t1 << "@" << endl;
  cout << t2 << "@" << endl;

  cout << SHOW(t1==t2) << endl;

  cout << SHOW(tokenizermisc::comparewithoutspace(s1,s2)) << endl;

}

void tokenizertest::example01(string& s)
{
  s=
"<gameD2>\n\
  <startcolor>.324,.1,.8928</startcolor>\n\
  <matrixD2>\n\
    <m>25</m>\n\
    <n>23</n>\n\
  </matrixD2>\n\
  <settings>\n\
    <walls>1</walls>\n\
    <pipes>0</pipes>\n\
  </settings>\n\
</gameD2>\
";
}

void tokenizertest::example02(string& s)
{
  s=
"<packet>\n\
  <header>garbage</header>\n\
  <data>\n\
  <field>\n\
    <name>EMPID</name>\n\
    <emp>Joe</emp>\n\
    <emp>Samantha</emp>\n\
    <emp>Key</emp>\n\
  </field>\n\
  <sig>ire83402l2992p2l2sl94202</sig>\n\
  </data>\n\
</packet>";
}

void tokenizertest::example03(string& s)
{
  s=
"<Warehouse>\
<Inventory>\
<obj>\
<Item>829 </Item>\
<Quantity>5 </Quantity>\
<Desc>Plastic sheet </Desc>\
<SN> 829984j892wwh</SN>\
<Cost> 10</Cost>\
</obj>\
<obj>\
<Item>88329 </Item>\
<Quantity> 1 </Quantity>\
<Desc> Van Guard  </Desc>\
<SN> </SN>\
<Cost> 2089498</Cost>\
</obj>\
<obj>\
<Item>2824 </Item>\
<Quantity> 1 </Quantity>\
<Desc> Dyson Orbital Weapon System  </Desc>\
<SN> 78914JAZY7K</SN>\
<Cost> 184000000</Cost>\
</obj>\
</Inventory>\
<Warehouse>\
";
}

void tokenizertest::test10()
{
  string s1;
  example01(s1);
  tokenizer tk(s1);

  cout << "Display string:" << endl;
  cout << tk << endl;
  cout << endl;
  cout << "atomize_next <startcolor>:" << endl;
  tk.reset();
  tk.atomize_next("<startcolor>");

  //cout << SHOW(tk()) << endl;

  liststringiterator i1(tk.seq);
  for (i1.reset(); !i1; ++i1)
    { cout << "*" << *i1 << "*" << endl; }

  cout << "atomize_next </startcolor>:" << endl;
  tk.atomize_next("</startcolor>");

  for (i1.reset(); !i1; ++i1)
    { cout << "*" << *i1 << "*" << endl; }
}

int tokenizertest::unittest01()
{
  {
    cout << "Test 1.1" << endln;
    string s1;
    example01(s1);
    tokenizer tk(s1);

    liststringi i1;
    liststringi i2;
    tk.reset();
    bool res;
    res=tk.atomize_next_tag(i1,i2,"startcolor");
    cout << "Atomize <startcolor> and </startcolor>" << endl;
    cout << SHOW(res) << endl;
    if (res==false)
      return 1;

    cout << "Read the tag." << endl;
    cout << "  check 3 strings." << endl;
    liststringi i3(i1);
    ++i3; ++i3;
    if (i3!=i2)
      return 1;
    --i3;
    cout << "tag data=\"" << *i3 << "\"" << endl;
    cout << "Writing a value back" << endl;
    *i3 = "0.1 0.1 0.1";

    tk.trim_and_prune();

    string s2 = (stringc)tk;
    cout << tk << endl;
  }

  {
    cout << "Test 1.2" << endln;
    string s1;
    example01(s1);
    tokenizer tk(s1);

    tk.reset();
    tk.atomize("<settings>");
    tk.atomize("<pipes>");

    cout << print(tk.seq,"*") << endln;
    string::size_type k1;

    tk.reset();
    assertreturnOS( tk.find(k1,"matrixD2") );
    cout << SHOW( *tk.current ) << " " << SHOW(k1) << " ";
    cout << SHOW(tk.current->substr(k1,8)) << endl;
    assertreturnOS(tk.current->substr(k1,8)=="matrixD2");
    assertreturnOS( tk.find(k1,"<n>",k1) );
    cout << SHOW( *tk.current ) << " " << SHOW(k1) << " ";
    cout << SHOW(tk.current->substr(k1,3)) << endl;
    assertreturnOS(tk.current->substr(k1,3)=="<n>");

    assertreturnOS( tk.find(k1,"<pipes>",k1) );
    cout << SHOW( *tk.current ) << " " << SHOW(k1) << endln;
  }

  {
    cout << "Test 1.3" << endln;
    string s1;
    example01(s1);
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();
    assertreturnOS( tl.scope("settings") );
    assertreturnOS( tl.scope("pipes") );

    tokenizerfind tf(tk,"<");
    tf.reset();
    for ( ; !tf; ++tf)
      cout << SHOW(tf()) << " " << SHOW(tf.index) << endln;

    cout << "find(\"matrixD2\",\"n\")" << endl;
    
    tk.reset();
    assertreturnOS( tf.find("<matrixD2>","<n>") );
    cout << SHOW(tf()) << " " << SHOW(tf.index) << endln;
    cout << SHOW( tf().substr(tf.index,3) ) << endl;

  }

  return 0;
}

int tokenizertest::unittest02()
{
  string s1;
  example01(s1);

  cout << "Testing tokenizerlocal::scope(stringc&);" << endl;
  cout << "Looking to see list of strings with *<matrixD2>* ... *</matrixD2>" << endl;

  {
    cout << "TEST 2.1" << endl;
    cout << "Atomizing tag" << endln;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();
    assertreturnOS( tl.scope("matrixD2") );

    liststringiterator i1(tk.seq);
    for (i1.reset(); !i1; ++i1)
      { cout << "*" << *i1 << "*" << endl; }
  }

  {
    cout << "TEST 2.2" << endl;
    cout << "Atomizing tag depth of 2." << endln;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset(); 
    assertreturnOS( tl.scope("matrixD2","m") );

    liststringiterator i1(tk.seq);
    for (i1.reset(); !i1; ++i1)
      { cout << "*" << *i1 << "*" << endl; }
  }

  {
    cout << "TEST 2.3" << endl;
    cout << "Reading and writing a tag at depth of 2." << endln;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset(); 

    string val;

    assertreturnOS( tl.read(val,"startcolor") );
    cout << "*" << SHOW(val) << "*" << endl;

    assertreturnOS( tl.read(val,"matrixD2","n") );
    cout << "*" << SHOW(val) << "*" << endl;

    cout << SHOW(*tl.i1) << SHOW(*tl.i2) << endl;
    cout << SHOW(*tl.ref) << endl;

    tl.reset();
    tl.writetag("i am starting to dislike working on the parser", "n");
    cout << SHOW(*tl.ref) << endl;

    cout << tk << endl;
  }

  {
    cout << "TEST 2.4" << endl;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();
    cout << "Erasing the <settings> tag." << endl;

    bool res;
    res = tl.scopesearch("settings");
    cout << SHOW(res) << endl;
    cout << "Did not expect to find settings as not atomized." << endl;
    assertreturnOS(res==false);
    tk.reset();
    res = tl.scope("settings");
    assertreturnOS(res==true);
    cout << SHOW(res) << " " << SHOW(*tl.i1) << " " << SHOW(*tl.i2) << endl;
    assertreturnOS(tl.erasetag("settings"));

    cout << tk << endl;
  }

  return 0;
}

int tokenizertest::unittest03()
{
  string s1;
  example02(s1);

  {
    cout << "TEST 3.1" << endl;
    tokenizer tk(s1);
    tokenizerlocal tl(tk);

    tk.reset();

    cout << "Listing the employees" << endln;
    cout << "  reading multiple <emp> string </emp>" << endln;

    string name;
    assertreturnOS(tl.read(name,"field","name"));
    cout << SHOW(name) << endl;
    string ei;
    list<string> names;
    for ( ; tl.read(ei,"emp"); )
      names.push_back(ei); 
    cout << "names:" << print(names," ") << endl;
    
    cout << "Edit by reading the whole list in, editing it";
    cout << " there and then re writing it." << endl;

    names.erase(find(names.begin(),names.end(),"Samantha"));

    names.push_back("Zelda");
    names.push_back("Ba");

    cout << "###" << print(names," ") << endl;


    liststringiterator namesi(names); 

    string s2;
    s2 = ("<name>" + name + "</name>");
    for ( ; !namesi; ++namesi)
    {
 assert(!namesi);
      s2 += ("<emp>"+*namesi+"</emp>");
    }
    tk.reset();
    tl.writetag(s2,"field");

    cout << tk << endl;
  }

  {
    string s0;
    example03(s0);
    cout << "TEST 3.2" << endl;
    cout << "Reading/writing multiple <obj><Item>...</obj>'s" << endln;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);
    string s2;
    tl.read(s2,"Inventory");

    list<InventoryObj> inventory;
    InventoryObj prod;

    tokenizer tk2(s2);
    tokenizerlocal tl2(tk2);
    tokenizerlocal tl3(tk2);
    string targ;
    spacertrim<> st;
    for ( tk2.reset(); tl2.scope("obj"); ++tl2)
    {
      // Atomic read.
      if (tl3.read(prod.item,"Item")==false)
        continue;
      st(prod.item);
      if (tl3.read(targ,"Quantity")==false)
        continue;
      stringfrom(prod.quantity,targ);
      if (tl3.read(prod.desc,"Desc")==false)
        continue;
      st(prod.desc);
      if (tl3.read(prod.sn,"SN")==false)
        continue;
      st(prod.sn);
      if (tl3.read(targ,"Cost")==false)
        continue;
      stringfrom(prod.cost,targ);
      
      inventory.push_back(prod);
    } 

    list<InventoryObj>::iterator k=inventory.begin();
    for ( ; k!=inventory.end(); ++k)
      cout << k->xml() << "*" << endl;

    cout << "Making edit" << endl;
    for ( k=inventory.begin(); k!=inventory.end(); ++k)
    {
//cout << SHOW(k->item) << "*" << endl;
      if (k->item=="88329")
        k->sn = "S298RT";
    }
    
    string result;
    for ( k=inventory.begin(); k!=inventory.end(); ++k)
      { result += k->xml(); }
    
    tl.reset();
    tl.writetag(result,"Inventory");

    cout << tl.ref << endl;
  }

  return 0;
}


stringc InventoryObj::xml()
{
  string x;

  x += stringtag(item,"Item");
  x += stringtag(quantity,"Quantity"); 
  x += stringtag(desc,"Desc");
  x += stringtag(sn,"SN");
  x += stringtag(cost,"Cost");

  return stringtag(x,"obj");
}

/*

// Extract - non specified number of string elements.
// Build a vector of strings for EMPID.
// Assumptions: 1st tag in packet->data->field scope
//   is <name> </name>
// read until no more <emp> records to read. 

Write a demo function that reads and re writes
 the number of emp's.

*/


int tokenizertest::unittest04()
{
  {
    string s0;
    example03(s0);

    cout << "TEST 4.1" << endl;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);

    bool res;

    res=tl.scope("Inventory");
    assertreturnOS(res);

    string SN;
    string Desc;

    tokenizerlocal tl2(tl);
    for ( ; !tl2; ++tl2 )
    {
      res=tl2.scope("obj");
      if (res)
      {
        tokenizerlocal tl3(tl2);
        if (tl3.read(SN,"SN"))
          { cout << SHOW3(SN) << endl; }
        tl3.reset(tl2);
        if (tl3.read(Desc,"Desc"))
          { cout << SHOW3(Desc) << endl; }
        
      }
    }  
  }
  {
    string s0;
    example03(s0);

    cout << "TEST 4.2" << endl;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);

    assertreturnOS( tl.scope("Inventory") );

    string SN;
    string Desc;

    tokenizerlocal tl2(tl);
    for ( ; tl2.scope("obj"); ++tl2 )
    {
      tokenizerlocal tl3(tl2);
      if (tl3.read(SN,"SN"))
        cout << SHOW3(SN) << endl;
cout << SHOW3(tl3.debug01()) << endl;
      tl3.reset();
cout << SHOW3(tl3.debug01()) << endl;
      if (tl3.read(Desc,"Desc"))
        cout << SHOW3(Desc) << endl;
    }  
  };

  {
    string s0;
    example03(s0);

    cout << "TEST 4.3" << endl;
    tokenizer tk(s0);
    tokenizerlocal tl(tk);

    assertreturnOS( tl.scope("Inventory") );

    string SN;
    string Desc;

    tokenizerlocalvar tl2(tl,"obj");
    for ( ; !tl2; ++tl2 )
    {
      tokenizerlocal tl3(tl2);
      if (tl3.read(SN,"SN"))
        cout << SHOW3(SN) << endl;
cout << SHOW3(tl3.debug01()) << endl;
      tl3.reset();
cout << SHOW3(tl3.debug01()) << endl;
      if (tl3.read(Desc,"Desc"))
        cout << SHOW3(Desc) << endl;
    }  
  };

  return 0;
}