Found a bug in Tokenzier

[Loaden]

Like thie code:

Code Select Expand

   Tokenizer tk;
   tk.InitFromBuffer(_T("15 + 25 * 2"));
   wxString s;
   while (tk.NotEOF())
   {
       s += tk.GetToken();
   }
The s Value is:

Code Select Expand

15+25*

Will lost the last char '2'.

This patch can fix it:

Code Select Expand


wxString Tokenizer::DoGetToken()
{
...
   else if (wxIsdigit(c))
   {
       // numbers
+        int count = 0;
       while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
       {
           MoveToNextChar();
+            ++count;
       }

-        if (IsEOF())
+        if (IsEOF() && count == 0)
            return wxEmptyString;
       }

Morten, your opinion?

[Loaden]

Maybe a better way, just comment these two lines!

Code Select Expand

        if (IsEOF())
            return wxEmptyString;
Even "IsEOF()" is true, "m_Buffer.Mid (start, m_TokenIndex - start);" can be also work!

Code Select Expand

    if (c == '_' || wxIsalpha(c))
    {
        // keywords, identifiers, etc.

        // operator== is cheaper than wxIsalnum, also MoveToNextChar already includes IsEOF
        while (    ( (c == '_') || (wxIsalnum(c)) )
               &&  MoveToNextChar() )
            c = CurrentChar(); // repeat

//        if (IsEOF())
//            return wxEmptyString;

        needReplace = true;
        str = m_Buffer.Mid(start, m_TokenIndex - start);
    }
#ifdef __WXMSW__ // This is a Windows only bug!
    else if (c == 178 || c == 179 || c == 185) // fetch ?and ?
    {
        str = c;
        MoveToNextChar();
    }
#endif
    else if (wxIsdigit(c))
    {
        // numbers
        while (NotEOF() && CharInString(CurrentChar(), _T("0123456789.abcdefABCDEFXxLl")))
        {
            MoveToNextChar();
        }

//        if (IsEOF())
//            return wxEmptyString;

        str = m_Buffer.Mid(start, m_TokenIndex - start);
        m_IsOperator = false;
    }