mirror of
https://github.com/stefanocasazza/ULib.git
synced 2025-09-28 19:05:55 +08:00
156 lines
4.1 KiB
C++
156 lines
4.1 KiB
C++
// test_tokenizer.cpp
|
|
|
|
#include <ulib/file.h>
|
|
#include <ulib/tokenizer.h>
|
|
#include <ulib/json/value.h>
|
|
|
|
int
|
|
U_EXPORT main (int argc, char* argv[], char* env[])
|
|
{
|
|
U_ULIB_INIT(argv);
|
|
|
|
U_TRACE(5, "main(%d)", argc)
|
|
|
|
UTokenizer t;
|
|
UString dati, y, z = U_STRING_FROM_CONSTANT("mnt mirror home stefano spool cross");
|
|
|
|
t.setData(z);
|
|
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("mnt") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("mirror") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("home") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("stefano") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("spool") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("cross") )
|
|
U_ASSERT( t.next(y,(bool*)0) == false )
|
|
|
|
t.setData(z);
|
|
t.setDelimiter(" \t\n");
|
|
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("mnt") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("mirror") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("home") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("stefano") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("spool") )
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("cross") )
|
|
U_ASSERT( t.next(y,(bool*)0) == false )
|
|
|
|
UString z1 = U_STRING_FROM_CONSTANT("\"spool cross\"");
|
|
|
|
t.setData(z1);
|
|
t.setDelimiter(0);
|
|
|
|
U_ASSERT( t.next(y,(bool*)0) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("spool cross") )
|
|
U_ASSERT( t.next(y,(bool*)0) == false )
|
|
|
|
UString z2 = U_STRING_FROM_CONSTANT("(pippo OR pluto) AND NOT (paperino AND paperone)");
|
|
|
|
t.setData(z2);
|
|
t.setDelimiter(0);
|
|
t.setGroup(U_CONSTANT_TO_PARAM("()"));
|
|
|
|
bool bgroup = false;
|
|
|
|
U_ASSERT( t.next(y,&bgroup) )
|
|
U_ASSERT( bgroup )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("pippo OR pluto") )
|
|
U_ASSERT( t.next(y,&bgroup) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("AND") )
|
|
U_ASSERT( bgroup == false )
|
|
U_ASSERT( t.next(y,&bgroup) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("NOT") )
|
|
U_ASSERT( bgroup == false )
|
|
U_ASSERT( t.next(y,&bgroup) )
|
|
U_ASSERT( y == U_STRING_FROM_CONSTANT("paperino AND paperone") )
|
|
U_ASSERT( bgroup )
|
|
U_ASSERT( t.next(y,(bool*)0) == false )
|
|
|
|
t.setGroup(0);
|
|
t.setDelimiter(0);
|
|
t.setSkipTagXML(true);
|
|
|
|
UString filename;
|
|
|
|
while (cin >> filename)
|
|
{
|
|
dati = UFile::contentOf(filename);
|
|
|
|
t.setData(dati);
|
|
|
|
while (t.next(y,(bool*)0)) cout << y << "\n";
|
|
}
|
|
|
|
if (argv[1])
|
|
{
|
|
y.clear();
|
|
|
|
dati = UFile::contentOf(UString(argv[1]));
|
|
|
|
t.setData(dati);
|
|
t.setGroup(U_CONSTANT_TO_PARAM("<%%>"));
|
|
|
|
while (t.next(y,(bool*)0))
|
|
{
|
|
cout << "------------------------------------------\n";
|
|
cout << y;
|
|
cout << "------------------------------------------\n";
|
|
|
|
uint32_t distance = t.getDistance(),
|
|
pos = U_STRING_FIND(dati, distance, "<%");
|
|
|
|
if (pos == U_NOT_FOUND) pos = dati.size();
|
|
|
|
cout << "------------------------------------------\n";
|
|
cout << dati.substr(distance, pos - distance);
|
|
cout << "------------------------------------------\n";
|
|
|
|
t.setDistance(pos);
|
|
}
|
|
}
|
|
|
|
y.clear();
|
|
|
|
t.setData(U_STRING_FROM_CONSTANT(" ( $QUERY_STRING = 'submitted' ) "));
|
|
|
|
while (t.getTokenId(0) > 0) {}
|
|
|
|
U_ASSERT( t.getTokenId(0) == 0 )
|
|
|
|
t.setData(U_STRING_FROM_CONSTANT(" ( ${QUERY_STRING} != submitted ) "));
|
|
|
|
while (t.getTokenId(0) > 0) {}
|
|
|
|
U_ASSERT( t.getTokenId(0) == 0 )
|
|
|
|
t.setData(U_STRING_FROM_CONSTANT("!!!.,;'?pippo.,;'?!!!"));
|
|
|
|
t.setAvoidPunctuation(true);
|
|
|
|
bool result = t.next(y,(bool*)0);
|
|
|
|
U_ASSERT( result )
|
|
|
|
result = (y == U_STRING_FROM_CONSTANT("pippo"));
|
|
|
|
U_ASSERT( result )
|
|
|
|
result = t.next(y,(bool*)0);
|
|
|
|
U_ASSERT( result == false )
|
|
|
|
y.clear();
|
|
}
|