improved handling of word commands if puntuation is following the word, preserved spaces at start of comment line

Examples/test-suite/doxygen_misc_constructs.h

@@ -69,6 +69,26 @@ void backslashC()
  * <pre>
  * ['retVal < 10', 'g_counter == 23 && g_mode & 3']
  *</pre>
+ *
+ * Both words should be emphasized \b isystem.connect.
+ * But not the last period. For \b example, comma should not be emphasized.
+ * Similar \b for: double colon.
+ *
+ * Spaces at the start of line should be taken into account:
+ * @param id used as prefix in log
+ *     statements. The default value is empty string, which is OK if
+ *     there is only one app. instance. Example:
+ *     <pre>
+ *         ctrl.setBP("func1");
+ *     </pre>
+ *     If we set the id to \c main_, we get:
+ *     <pre>
+ *         main_ctrl.setBP("func1");
+ *     </pre>
+ *
+ * @param fileName name of the log file
  */
-void cycle()
+void cycle(int id, char *fileName)
 {}
+
+

Examples/test-suite/java/doxygen_misc_constructs_runme.java

@@ -154,13 +154,29 @@ public class doxygen_misc_constructs_runme {
                 " @see MyClass#fun(char,float)\n" +
     		"");
 
-   wantedComments.put("doxygen_misc_constructs.doxygen_misc_constructs.cycle()",
+   wantedComments.put("doxygen_misc_constructs.doxygen_misc_constructs.cycle(int, java.lang.String)",
     		" The next line contains expression:\n" +
                 " <pre>\n" +
     		" ['retVal &lt; 10', 'g_counter == 23 &amp;&amp; g_mode &amp; 3']\n" +
                 " </pre>\n" +
     		"\n" +
-    		"");
+                " Both words should be emphasized <b>isystem.connect</b>.\n" +
+                " But not the last period. For <b>example</b>, comma should not be emphasized.\n" +
+                " Similar <b>for</b>: double colon.\n" +
+    		"\n" +
+                " Spaces at the start of line should be taken into account:\n" +
+                " @param id used as prefix in log\n" +
+                "     statements. The default value is empty string, which is OK if\n" +
+                "     there is only one app. instance. Example:\n" +
+                "     <pre>\n" +
+                "         ctrl.setBP(\"func1\");\n" +
+                "     </pre>\n" +
+                "     If we set the id to <code>main_</code>, we get:\n" +
+                "     <pre>\n" +
+                "         main_ctrl.setBP(\"func1\");\n" +
+                "     </pre>\n" +
+                "\n" +
+                " @param fileName name of the log file\n");
  
     
     // and ask the parser to check comments for us

Examples/test-suite/java/doxygen_translate_all_tags_runme.java

@@ -52,7 +52,7 @@ public class doxygen_translate_all_tags_runme {
     		" and detailed description of some thing \n");
     
     wantedComments.put("doxygen_translate_all_tags.doxygen_translate_all_tags.func03(int)",
-                       " Comment for <b>func03().</b>\n" +
+                       " Comment for <b>func03()</b>.\n" +
                        " <i>italicword </i>\n" +
     		" <i>emphazedWord </i>\n" +
                 " @ example someFile.txt\n" +
@@ -87,7 +87,7 @@ public class doxygen_translate_all_tags_runme {
     		" describing invariant. \n");
     
     wantedComments.put("doxygen_translate_all_tags.doxygen_translate_all_tags.func06(int)",
-                       " Comment for <b>func06().</b>\n" +
+                       " Comment for <b>func06()</b>.\n" +
                        " This will only appear in LATeX \n" +
     		" <ul> \n" +
     		" <li>Some unordered list \n" +
@@ -98,7 +98,7 @@ public class doxygen_translate_all_tags_runme {
     		" This will only appear in man\n");
     
     wantedComments.put("doxygen_translate_all_tags.doxygen_translate_all_tags.func07(int)",
-                       " Comment for <b>func07().</b>\n" +
+                       " Comment for <b>func07()</b>.\n" +
                        " Note: Here \n" +
     		" is the note! \n" +
     		" This is an overloaded member function, provided for convenience. It differs from the above function only in what argument(s) it accepts.\n" +

Examples/test-suite/python/autodoc_runme.py

@@ -1,14 +1,12 @@
 from autodoc import *
+import commentVerifier
 
-def check(got, expected):
-  if expected != got:
-    raise RuntimeError("\n" + "Expected: [" + str(expected) + "]\n" + "Got     : [" + str(got) + "]")
 
-check(A.__doc__, "Proxy of C++ A class")
+commentVerifier.check(A.__doc__, "Proxy of C++ A class")
-check(A.funk.__doc__, "just a string")
+commentVerifier.check(A.funk.__doc__, "just a string")
-check(A.func0.__doc__, "func0(self, arg2, hello) -> int")
+commentVerifier.check(A.func0.__doc__, "func0(self, arg2, hello) -> int")
-check(A.func1.__doc__, "func1(A self, short arg2, Tuple hello) -> int")
+commentVerifier.check(A.func1.__doc__, "func1(A self, short arg2, Tuple hello) -> int")
-check(A.func2.__doc__,  "\n"
+commentVerifier.check(A.func2.__doc__,  "\n"
 "        func2(self, arg2, hello) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -17,7 +15,7 @@ check(A.func2.__doc__,  "\n"
 "\n"
 "        "
 )
-check(A.func3.__doc__,  "\n"
+commentVerifier.check(A.func3.__doc__,  "\n"
 "        func3(A self, short arg2, Tuple hello) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -27,17 +25,17 @@ check(A.func3.__doc__,  "\n"
 "        "
 )
 
-check(A.func0default.__doc__, "\n"
+commentVerifier.check(A.func0default.__doc__, "\n"
 "        func0default(self, e, arg3, hello, f=2) -> int\n"
 "        func0default(self, e, arg3, hello) -> int\n"
 "        "
 )
-check(A.func1default.__doc__, "\n"
+commentVerifier.check(A.func1default.__doc__, "\n"
 "        func1default(A self, A e, short arg3, Tuple hello, double f=2) -> int\n"
 "        func1default(A self, A e, short arg3, Tuple hello) -> int\n"
 "        "
 )
-check(A.func2default.__doc__,  "\n"
+commentVerifier.check(A.func2default.__doc__,  "\n"
 "        func2default(self, e, arg3, hello, f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -55,7 +53,7 @@ check(A.func2default.__doc__,  "\n"
 "\n"
 "        "
 )
-check(A.func3default.__doc__,  "\n"
+commentVerifier.check(A.func3default.__doc__,  "\n"
 "        func3default(A self, A e, short arg3, Tuple hello, double f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -74,17 +72,17 @@ check(A.func3default.__doc__,  "\n"
 "        "
 )
 
-check(A.func0static.__doc__, "\n"
+commentVerifier.check(A.func0static.__doc__, "\n"
 "        func0static(e, arg2, hello, f=2) -> int\n"
 "        func0static(e, arg2, hello) -> int\n"
 "        "
 )
-check(A.func1static.__doc__, "\n"
+commentVerifier.check(A.func1static.__doc__, "\n"
 "        func1static(A e, short arg2, Tuple hello, double f=2) -> int\n"
 "        func1static(A e, short arg2, Tuple hello) -> int\n"
 "        "
 )
-check(A.func2static.__doc__,  "\n"
+commentVerifier.check(A.func2static.__doc__,  "\n"
 "        func2static(e, arg2, hello, f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -102,7 +100,7 @@ check(A.func2static.__doc__,  "\n"
 "\n"
 "        "
 )
-check(A.func3static.__doc__,  "\n"
+commentVerifier.check(A.func3static.__doc__,  "\n"
 "        func3static(A e, short arg2, Tuple hello, double f=2) -> int\n"
 "\n"
 "        Parameters:\n"
@@ -121,16 +119,16 @@ check(A.func3static.__doc__,  "\n"
 "        "
 )
 
-check(A.variable_a.__doc__, "A_variable_a_get(self) -> int")
+commentVerifier.check(A.variable_a.__doc__, "A_variable_a_get(self) -> int")
-check(A.variable_b.__doc__, "A_variable_b_get(A self) -> int")
+commentVerifier.check(A.variable_b.__doc__, "A_variable_b_get(A self) -> int")
-check(A.variable_c.__doc__, "\n"
+commentVerifier.check(A.variable_c.__doc__, "\n"
 "A_variable_c_get(self) -> int\n"
 "\n"
 "Parameters:\n"
 "    self: A *\n"
 "\n"
 )
-check(A.variable_d.__doc__, "\n"
+commentVerifier.check(A.variable_d.__doc__, "\n"
 "A_variable_d_get(A self) -> int\n"
 "\n"
 "Parameters:\n"
@@ -138,10 +136,10 @@ check(A.variable_d.__doc__, "\n"
 "\n"
 )
 
-check(B.__doc__, "Proxy of C++ B class")
+commentVerifier.check(B.__doc__, "Proxy of C++ B class")
-check(C.__init__.__doc__, "__init__(self, a, b, h) -> C")
+commentVerifier.check(C.__init__.__doc__, "__init__(self, a, b, h) -> C")
-check(D.__init__.__doc__, "__init__(D self, int a, int b, Hola h) -> D")
+commentVerifier.check(D.__init__.__doc__, "__init__(D self, int a, int b, Hola h) -> D")
-check(E.__init__.__doc__, "\n"
+commentVerifier.check(E.__init__.__doc__, "\n"
 "        __init__(self, a, b, h) -> E\n"
 "\n"
 "        Parameters:\n"
@@ -151,7 +149,7 @@ check(E.__init__.__doc__, "\n"
 "\n"
 "        "
 )
-check(F.__init__.__doc__, "\n"
+commentVerifier.check(F.__init__.__doc__, "\n"
 "        __init__(F self, int a, int b, Hola h) -> F\n"
 "\n"
 "        Parameters:\n"
@@ -162,15 +160,15 @@ check(F.__init__.__doc__, "\n"
 "        "
 )
 
-check(B.funk.__doc__, "funk(B self, int c, int d) -> int")
+commentVerifier.check(B.funk.__doc__, "funk(B self, int c, int d) -> int")
-check(funk.__doc__, "funk(A e, short arg2, int c, int d) -> int")
+commentVerifier.check(funk.__doc__, "funk(A e, short arg2, int c, int d) -> int")
-check(funkdefaults.__doc__, "\n"
+commentVerifier.check(funkdefaults.__doc__, "\n"
 "  funkdefaults(A e, short arg2, int c, int d, double f=2) -> int\n"
 "  funkdefaults(A e, short arg2, int c, int d) -> int\n"
 "  "
 )
 
-check(func_input.__doc__, "func_input(int * INPUT) -> int")
+commentVerifier.check(func_input.__doc__, "func_input(int * INPUT) -> int")
-check(func_output.__doc__, "func_output() -> int")
+commentVerifier.check(func_output.__doc__, "func_output() -> int")
-check(func_inout.__doc__, "func_inout(int * INOUT) -> int")
+commentVerifier.check(func_inout.__doc__, "func_inout(int * INOUT) -> int")
-check(banana.__doc__, "banana(S a, S b, int c, Integer d)")
+commentVerifier.check(banana.__doc__, "banana(S a, S b, int c, Integer d)")

Examples/test-suite/python/doxygen_misc_constructs_runme.py

@@ -113,3 +113,32 @@ commentVerifier.check(doxygen_misc_constructs.backslashC.__doc__,
    float)
   """
 )
+
+
+commentVerifier.check(doxygen_misc_constructs.cycle.__doc__,
+  r"""
+  The next line contains expression:
+
+  ['retVal < 10', 'g_counter == 23 && g_mode & 3']
+
+
+  Both words should be emphasized __isystem.connect__.
+  But not the last period. For __example__, comma should not be emphasized.
+  Similar __for__: double colon.
+
+  Spaces at the start of line should be taken into account:
+  Arguments:
+    id (int) -- used as prefix in log
+      statements. The default value is empty string, which is OK if
+      there is only one app. instance. Example:
+      
+          ctrl.setBP("func1");
+      
+      If we set the id to 'main_', we get:
+      
+          main_ctrl.setBP("func1");
+      
+
+    fileName (char *) -- name of the log file
+  """
+);

Examples/test-suite/python/doxygen_translate_all_tags_runme.py

@@ -67,7 +67,7 @@ r"""
 
 commentVerifier.check(doxygen_translate_all_tags.func03.__doc__,
 r"""
-  Comment for __func03().__
+  Comment for __func03()__.
 
 
 
@@ -125,9 +125,9 @@ r"""
       Nested condition text
     }Else if: THIRDCONDITION {
       The third condition text
-  }Else:  {The last text block
+    }Else:  {    The last text block
     }
-  }Else:  {Second part of comment
+  }Else:  {  Second part of comment
     If: CONDITION {
       Second part extended
     }
@@ -155,7 +155,7 @@ r"""
 
 commentVerifier.check(doxygen_translate_all_tags.func06.__doc__,
 r"""
-  Comment for __func06().__
+  Comment for __func06()__.
 
 
 
@@ -194,7 +194,7 @@ r"""
 
 commentVerifier.check(doxygen_translate_all_tags.func07.__doc__,
 r"""
-  Comment for __func07().__
+  Comment for __func07()__.
 
 
 

Examples/test-suite/python/doxygen_translate_runme.py

@@ -45,9 +45,9 @@ r"""
       Nested condition text
     }Else if: THIRDCONDITION {
       The third condition text
-  }Else:  {The last text block
+    }Else:  {    The last text block
     }
-  }Else:  {Second part of comment
+  }Else:  {  Second part of comment
     If: CONDITION {
       Second part extended
     }

Source/DoxygenTranslator/src/DoxygenParser.cpp

@@ -223,13 +223,7 @@ std::string DoxygenParser::getNextWord() {
     if (!tokenStr.empty()) {
       return tokenStr;
     }
-	} /* else if (nextToken.m_tokenType == END_LINE) {
+  }
-	    // this handles cases when command is the last item in line, for example:
-	    // * This method returns line number \c
-	    // * relative to paragraph.
-        m_tokenListIt++;
-        return getNextWord();
-	} */
 
   return "";
 }
@@ -1264,32 +1258,69 @@ void DoxygenParser::tokenizeDoxygenComment(const std::string &doxygenComment,
     const string &line = *it;
     size_t pos = line.find_first_not_of(" \t");
 
-    // skip sequences of '*', '/', and '!' of any length
-    while (pos != string::npos  &&  isStartOfDoxyCommentChar(line[pos])) {
-      pos++;
-    }
-
     if (pos == string::npos) {
       m_tokenList.push_back(Token(END_LINE, "\n"));
       continue;
     }
 
+    // skip sequences of '*', '/', and '!' of any length
+    bool isStartOfCommentLineCharFound = false;
+    while (pos < line.size()  &&  isStartOfDoxyCommentChar(line[pos])) {
+      pos++;
+      isStartOfCommentLineCharFound = true;
+    }
+
+    if (pos == line.size()) {
+      m_tokenList.push_back(Token(END_LINE, "\n"));
+      continue;
+    }
+
+    // if 'isStartOfCommentLineCharFound' then preserve leading spaces, so
+    // ' *    comment' gets translated to ' *    comment', not ' * comment'
+    // This is important to keep formatting for comments translated to Python.
+    if (isStartOfCommentLineCharFound  &&  line[pos] == ' ') {
+      pos++; // points to char after ' * '
+      if (pos == line.size()) {
+        m_tokenList.push_back(Token(END_LINE, "\n"));
+        continue;
+      }
+    }
+
     // line[pos] may be ' \t' or start of word, it there was no '*', '/' or '!'
     // at beginning of the line. Make sure it points to start of the first word
     // in the line.
-    pos = line.find_first_not_of(" \t", pos);
+    size_t firstWordPos = line.find_first_not_of(" \t", pos);
-    if (pos == string::npos) {
+    if (firstWordPos == string::npos) {
       m_tokenList.push_back(Token(END_LINE, "\n"));
       continue;
     }
 
+    if (isStartOfCommentLineCharFound  &&  firstWordPos > pos) {
+      m_tokenList.push_back(Token(PLAINSTRING, line.substr(pos, firstWordPos - pos)));
+    }
+
+    pos = firstWordPos;
+
     while (pos != string::npos) {
       // find the end of the word
       size_t doxyCmdOrHtmlTagPos = line.find_first_of("\\@<>&\" \t", pos);
       if (doxyCmdOrHtmlTagPos != pos) {
         // plain text found
-        m_tokenList.push_back(Token(PLAINSTRING,
+        // if the last char is punctuation, make it a separate word, otherwise
-                              line.substr(pos, doxyCmdOrHtmlTagPos - pos)));
+        // it may be included with word also when not appropriate, for example:
+        //   colors are \b red, green, and blue --> colors are <b>red,</b> green, and blue
+        // instead of (comma not bold):
+        //   colors are \b red, green, and blue --> colors are <b>red</b>, green, and blue
+        // In Python it looks even worse:
+        //   colors are \b red, green, and blue --> colors are 'red,' green, and blue
+        string text = line.substr(pos, doxyCmdOrHtmlTagPos - pos);
+        string punctuations(".,:");
+        if (!text.empty()  &&  punctuations.find(text[text.size() - 1]) != string::npos) {
+          m_tokenList.push_back(Token(PLAINSTRING, text.substr(0, text.size() - 1)));
+          m_tokenList.push_back(Token(PLAINSTRING, text.substr(text.size() - 1)));
+        } else {
+          m_tokenList.push_back(Token(PLAINSTRING, text));
+        }
       }
 
       pos = doxyCmdOrHtmlTagPos;