For example, when parsing this statement:
#define _NO_DEBUG_PLACEHOLDER _No_debug_placeholder = _No_debug_placeholder()
See the source code in parserthread.cpp
void ParserThread::HandleDefines()
{
wxString filename;
int lineNr = m_Tokenizer.GetLineNumber();
wxString token = m_Tokenizer.GetToken(); // read the token after #define
m_Str.Clear();
// now token holds something like:
// BLAH_BLAH
if (!token.IsEmpty())
{
// skip the rest of the #define
wxString defVal = token + m_Tokenizer.ReadToEOL();
wxString para(_T(""));
int start = defVal.Find('(');
int end = defVal.Find(')');
TRACE(_T("HandleDefines() : Saving nesting level: %d,%d"), start, end);
// make sure preprocessor definitions are not going under namespaces or classes!
if (start != wxNOT_FOUND && end != wxNOT_FOUND)
{
para = defVal.Mid(start, end-start+1);
m_Str = defVal.Mid(end + 1);
m_Str.Trim(false);
}
else
{
m_Str = defVal.substr(token.length());
m_Str.Trim(false);
//defVal = _T("");
}
Token* oldParent = m_pLastParent;
m_pLastParent = 0L;
DoAddToken(tkPreprocessor, token, lineNr, lineNr, m_Tokenizer.GetLineNumber(), para, false, true);
m_pLastParent = oldParent;
m_Str.Clear();
}
}
The "()" at the end of the line was mistakenly regarded as the "parameter" of the define, like below:
#define _NO_DEBUG_PLACEHOLDER() XXXXXX
Patch for it:
thank you.now, i'm at home and view this forum in my gphone. i will test it when i am at work.
Happy Chinese New Year to every fourmers!
it is better to replace
m_Str << m_Tokenizer.ReadToEOL();
with
if (!m_Str.IsEmpty())
m_Str << _T(" ") << m_Tokenizer.ReadToEOL();
we need a space between two word here.
Quote from: blueshake on February 18, 2010, 07:59:30 AM
if (!m_Str.IsEmpty())
m_Str << _T(" ") << m_Tokenizer.ReadToEOL();
we need a space between two word here.
Are you sure? Because in the case
m_Str.IsEmpty(),
m_Tokenizer.ReadToEOL() is not being called. Didn't you mean something like:
if (!m_Str.IsEmpty())
m_Str << _T(" "):
m_Str << m_Tokenizer.ReadToEOL();
???
haha,you are right. :D
if (!m_Str.IsEmpty())
m_Str << _T(" ");
m_Str << m_Tokenizer.ReadToEOL();
Quote from: blueshake on February 18, 2010, 09:29:12 AM
haha,you are right. :D
Probably this or the other patch of yours makes the total number of tokens to decrease from 138000 to 119000 in my C::B project. This doesn't look good to me.
Quote from: MortenMacFly on February 18, 2010, 03:32:46 PM
This doesn't look good to me.
It seems that all "simple" defines like:
#define BOOST_ALIGNED_STORAGE_HPP
are lost now.
this one cause the issue.I will check into it.
Ok,it seems I have solve this.I will update it when I get my machine.
new patch:
I just add a line compare here.because for codes:
#define BOOST_ALIGNED_STORAGE_HPP
int main() {}
when we do this,
wxString para = m_Tokenizer.GetToken();
at this time ,m_Tokenizer will return int ,then we do this
m_Str << m_Tokenizer.ReadToEOL();
it will eat the whole line codes
int main()
that is why the tokens number are lost.
BTW. @morten
can we print the whole project's tokens name to the file??so when we make patch,when the tokens numbers are wrong.we can use diff tool to check whick tokens are lost.
Another update.
remove the compile warning.
forget to do this in last patch.
Quoteif (!m_Str.IsEmpty())
m_Str << _T(" ");
m_Str << m_Tokenizer.ReadToEOL();
Index: src/plugins/codecompletion/parser/parserthread.cpp
===================================================================
--- src/plugins/codecompletion/parser/parserthread.cpp (revision 6169)
+++ src/plugins/codecompletion/parser/parserthread.cpp (working copy)
@@ -1123,8 +1123,8 @@
void ParserThread::HandleDefines()
{
- wxString filename;
- int lineNr = m_Tokenizer.GetLineNumber();
+ //wxString filename;
+ size_t lineNr = m_Tokenizer.GetLineNumber();
wxString token = m_Tokenizer.GetToken(); // read the token after #define
m_Str.Clear();
// now token holds something like:
@@ -1132,31 +1132,30 @@
if (!token.IsEmpty())
{
// skip the rest of the #define
- wxString defVal = token + m_Tokenizer.ReadToEOL();
- wxString para(_T(""));
- int start = defVal.Find('(');
- int end = defVal.Find(')');
-
+ wxString para = m_Tokenizer.GetToken();
TRACE(_T("HandleDefines() : Saving nesting level: %d,%d"), start, end);
-
- // make sure preprocessor definitions are not going under namespaces or classes!
- if (start != wxNOT_FOUND && end != wxNOT_FOUND)
+ if (lineNr == m_Tokenizer.GetLineNumber())
{
- para = defVal.Mid(start, end-start+1);
- m_Str = defVal.Mid(end + 1);
- m_Str.Trim(false);
+ if (para.IsEmpty() || para.GetChar(0) != '(') //check if contain the open brace
+ {
+ m_Str = para;
+ para = wxEmptyString;
+ }
+ if (!m_Str.IsEmpty())
+ m_Str << _T(" ");
+ m_Str << m_Tokenizer.ReadToEOL();
}
else
{
- m_Str = defVal.substr(token.length());
- m_Str.Trim(false);
- //defVal = _T("");
+ m_Tokenizer.UngetToken();
+ para = wxEmptyString;
+ m_Str << m_Tokenizer.ReadToEOL();
}
Token* oldParent = m_pLastParent;
m_pLastParent = 0L;
DoAddToken(tkPreprocessor, token, lineNr, lineNr, m_Tokenizer.GetLineNumber(), para, false, true);
m_pLastParent = oldParent;
- m_Str.Clear();
+ //m_Str.Clear();
}
}
Quote from: blueshake on February 19, 2010, 04:16:12 AM
can we print the whole project's tokens name to the file??so when we make patch,when the tokens numbers are wrong.we can use diff tool to check whick tokens are lost. [/b]
Sure. Using the token debug window (CTRL + double-click on a token). There you'll find a save button that allows to save the file list as well as the whole token tree to a file. That's why I could tell you what's missing btw... ;-)
But be careful: Creating the token tree can take a long time! C::B seems frozen then but it's not. Just be patient.
Quote from: blueshake on February 19, 2010, 04:16:12 AM
at this time ,m_Tokenizer will return int ,then we do this
m_Str << m_Tokenizer.ReadToEOL();
It depends on the settings of the Tokenizer (what to skip) IMHO.