implemented translation of HTML tags - they are handled as a special class of commands

Examples/test-suite/java/doxygen_translate_runme.java

@@ -80,7 +80,7 @@ public class doxygen_translate_runme {
     		" <li>With lots of items</li>\n" +
     		" <li>lots of lots of items</li>\n" +
     		" \n" +
-    		" </li></ul> \n" +
+    		" </ul> \n" +
     		" \n" +
     		" {@link someMember Some description follows }\n" +
     		" \n" +

Source/DoxygenTranslator/src/DoxygenCommands.h

@@ -83,19 +83,19 @@ const int commandUniquesSize = sizeof(commandUniques) / sizeof(*commandUniques);
 // Other commands are left intact, but '<' and '> are replaced with entities in HTML
 // output. So <varName> appears as &lt;varName&gt; in HTML output. The same
 // behavior must be repeated by SWIG. See Doxygen doc for the list of commands.
-// '<' and '>' are used to differentiate HTML commands from doxygen commands.
+// '<' is prepended to distinguish HTML tags from Doxygen commands.
-const char *htmlCommands[] = {
+const char *commandHtml[] = {
-  "<A>", "<B>", "<BLOCKQUOTE>", "<BODY>", "<BR>", "<CENTER>", "<CAPTION>", "<CODE>", "<DFN>", "<DFN>",
+  "<a", "<b", "<blockquote", "<body", "<br", "<center", "<caption", "<code", "<dfn",
-  "<DIV>", "<DL>", "<DT>", "<EM>", "<FORM>", "<HR>", "<H1>", "<H2>", "<H3>", "<I>", "<INPUT>", "<IMG>",
+  "<div", "<dl", "<dt", "<em", "<form", "<hr", "<h1", "<h2", "<h3", "<i", "<input", "<img",
-  "<LI>", "<META>", "<MULTICOL>", "<OL>", "<P>", "<PRE>", "<SMALL>", "<SPAN>", "<STRONG>",
+  "<li", "<meta", "<multicol", "<ol", "<p", "<pre", "<small", "<span", "<strong",
-  "<SUB>", "<SUP>", "<TABLE>", "<TD>", "<TH>", "<TR>", "<TT>", "<KBD>", "<UL>", "<VAR>"
+  "<sub", "<sup", "<table", "<td", "<th", "<tr", "<tt", "<kbd", "<ul", "<var"
 };
 
-const int htmlCommandsSize = sizeof(htmlCommands) / sizeof(*htmlCommands);
+const int commandHtmlSize = sizeof(commandHtml) / sizeof(*commandHtml);
 
 // Only entities which are translatable to plain text are used here. Others
 // are copied unchanged to output.
-const char *htmlEntities[] = { "&copy", "&trade", "&reg", // (C), (TM), (R)
+const char *commandHtmlEntities[] = { "&copy", "&trade", "&reg", // (C), (TM), (R)
 "&lt",  // less-than symbol
 "&gt",  // greater-than symbol
 "&amp", // ampersand
@@ -118,6 +118,6 @@ const char *htmlEntities[] = { "&copy", "&trade", "&reg", // (C), (TM), (R)
 "&rarr" // -->
 };
 
-const int htmlEntitiesSize = sizeof(htmlEntities) / sizeof(*htmlEntities);
+const int commandHtmlEntitiesSize = sizeof(commandHtmlEntities) / sizeof(*commandHtmlEntities);
 
 #endif

Source/DoxygenTranslator/src/DoxygenParser.cpp

@@ -27,7 +27,7 @@ DoxygenParser::DoxyCommandsMap DoxygenParser::doxygenCommands;
 std::set<std::string> DoxygenParser::doxygenSectionIndicators;
 
 const int TOKENSPERLINE = 8;    //change this to change the printing behaviour of the token list
-
+const std::string END_HTML_TAG_MARK("/");
 
 DoxygenParser::DoxygenParser(bool noisy) : noisy(noisy)
 {
@@ -76,11 +76,11 @@ void DoxygenParser::fillTables() {
     for (int i = 0; i < commandUniquesSize; i++)
         doxygenCommands[commandUniques[i]] = COMMANDUNIQUE;
 
-    for (int i = 0; i < htmlCommandsSize; i++)
+    for (int i = 0; i < commandHtmlSize; i++)
-        doxygenCommands[htmlCommands[i]] = COMMANDUNIQUE;
+        doxygenCommands[commandHtml[i]] = COMMAND_HTML;
 
-    for (int i = 0; i < commandUniquesSize; i++)
+    for (int i = 0; i < commandHtmlEntitiesSize; i++)
-        doxygenCommands[commandUniques[i]] = COMMANDUNIQUE;
+        doxygenCommands[commandHtmlEntities[i]] = COMMAND_HTML_ENTITY;
 
     // fill section indicators command set
     for (int i = 0; i < sectionIndicatorsSize; i++)
@@ -575,6 +575,32 @@ int DoxygenParser::addCommandErrorThrow(const std::string &theCommand,
 }
 
 
+int DoxygenParser::addCommandHtml(const std::string &theCommand,
+                                      const TokenList &,
+                                      DoxygenEntityList &doxyList)
+{
+    if (noisy)
+        cout << "Parsing " << theCommand << endl;
+
+    std::string htmlTagArgs = getNextWord();
+    doxyList.push_back(DoxygenEntity(theCommand, htmlTagArgs));
+    return 1;
+}
+
+
+int DoxygenParser::addCommandHtmlEntity(const std::string &theCommand,
+                                             const TokenList &,
+                                             DoxygenEntityList &doxyList)
+{
+    if (noisy)
+        cout << "Parsing " << theCommand << endl;
+
+    DoxygenEntityList aNewList;
+    doxyList.push_back(DoxygenEntity(theCommand, aNewList));
+    return 1;
+}
+
+
 int DoxygenParser::addCommandUnique(const std::string &theCommand,
                                          const TokenList &tokList,
                                          DoxygenEntityList &doxyList) {
@@ -893,6 +919,10 @@ int DoxygenParser::addCommand(const std::string &commandString,
 			return addCommandErrorThrow(theCommand, tokList, doxyList);
 		case COMMANDUNIQUE:
 			return addCommandUnique(theCommand, tokList, doxyList);
+        case COMMAND_HTML:
+            return addCommandHtml(theCommand, tokList, doxyList);
+        case COMMAND_HTML_ENTITY:
+            return addCommandHtmlEntity(theCommand, tokList, doxyList);
 	}
 	return 0;
 }
@@ -1068,10 +1098,11 @@ bool DoxygenParser::isStartOfDoxyCommentChar(char c)
 }
 
 
-void DoxygenParser::addDoxyCommand(DoxygenParser::TokenList &tokList,
+bool DoxygenParser::addDoxyCommand(DoxygenParser::TokenList &tokList,
                                         const std::string &cmd) {
     if (findCommand(cmd)) {
         tokList.push_back(Token(COMMAND, cmd));
+        return true;
     } else {
         // Unknown commands are ignored, because they are
         // also ignored by Doxygen - see test doxygen_misc_constructs.h, f. backslashB().
@@ -1079,6 +1110,8 @@ void DoxygenParser::addDoxyCommand(DoxygenParser::TokenList &tokList,
         // the line below to put unknown commands to output.
         // tokList.push_back(Token(PLAINSTRING, cmd));
     }
+
+    return false;
 }
 
 
@@ -1161,16 +1194,53 @@ size_t DoxygenParser::processNormalComment(size_t pos, const std::string &line)
     } break;
 
     case '<': { // process html commands
+      bool isEndHtmlTag = false;
+      pos++;
+      if (line.size() > pos &&  line[pos] == '/') {
+          isEndHtmlTag = true;
+          pos++;
+      }
 
-      size_t endHtmlPos = line.find_first_of("\t >", pos + 1);
+      size_t endHtmlPos = line.find_first_of("\t\n >", pos);
-      if (endHtmlPos != string::npos) {
+
-        // will push plain string Token. If the command is not HTML supported by
+      // prepend '<' to distinguish HTML tags from doxygen commands
-        // Doxygen, < and > will be replaced by HTML entities &lt; and &gt; respectively,
+      string cmd = line.substr(pos, endHtmlPos - pos);
+      pos = endHtmlPos;
+
+      if (addDoxyCommand(m_tokenList, '<' + cmd)) {
+          // it is a valid HTML command
+          if (line[pos] != '>') { // it should be HTML tag with args,
+                                   // for example <A ...>, <IMG ...>, ...
+              if (isEndHtmlTag) {
+                  m_tokenListIt = m_tokenList.end();
+                  printListError(WARN_DOXYGEN_COMMAND_ERROR, "Illegal end HTML tag without '>' found! Tag: " + cmd);
+              }
+              endHtmlPos = line.find(">", pos);
+              if (endHtmlPos == string::npos) {
+                  m_tokenListIt = m_tokenList.end();
+                  printListError(WARN_DOXYGEN_COMMAND_ERROR, "HTML tag without '>' found! Tag: " + cmd);
+              }
+              // add args of HTML command, like link URL, image URL, ...
+              m_tokenList.push_back(Token(PLAINSTRING,
+                                    line.substr(pos, endHtmlPos - pos)));
+          } else {
+              if (isEndHtmlTag) {
+                // it is a simple tag, so push empty string
+                m_tokenList.push_back(Token(PLAINSTRING, END_HTML_TAG_MARK));
+              } else {
+                // it is a simple tag, so push empty string
+                m_tokenList.push_back(Token(PLAINSTRING, ""));
+              }
+          }
+      } else {
+        // the command is not HTML supported by Doxygen, < and > will be
+        // replaced by HTML entities &lt; and &gt; respectively,
         // but only if 'htmlOnly' flag == false. The flag is set/reset by \htmlonly \verbatim,
         // \endhtmlonly \endverbatim Doxygen commands.
-        // handleHTMLCommand(line.substr(pos + 1), endHtmlPos - pos - 1);
+        m_tokenList.push_back(Token(PLAINSTRING, "&lt;"));
+        m_tokenList.push_back(Token(PLAINSTRING, cmd));
       }
-      pos = endHtmlPos;
+      pos++;
     } break;
 
     case '&': { // process HTML entities
@@ -1190,6 +1260,7 @@ size_t DoxygenParser::processNormalComment(size_t pos, const std::string &line)
     }
     break;
     default:
+      m_tokenListIt = m_tokenList.end();
       printListError(WARN_DOXYGEN_COMMAND_ERROR, "Unknown special character: " + line[pos]);
     }
 

Source/DoxygenTranslator/src/DoxygenParser.h

@@ -33,6 +33,8 @@ private:
     COMMANDOWORD,
     COMMANDERRORTHROW,
     COMMANDUNIQUE,
+    COMMAND_HTML,
+    COMMAND_HTML_ENTITY,
     END_LINE,
     PARAGRAPH_END,
     PLAINSTRING,
@@ -310,6 +312,14 @@ private:
                            const TokenList &tokList,
                            DoxygenEntityList &doxyList);
 
+  int addCommandHtml(const std::string &theCommand,
+                     const TokenList &tokList,
+                     DoxygenEntityList &doxyList);
+
+  int addCommandHtmlEntity(const std::string &theCommand,
+                           const TokenList &tokList,
+                           DoxygenEntityList &doxyList);
+
   /*
    *Adds the unique commands- different process for each unique command
    */
@@ -351,7 +361,7 @@ private:
 
   StringVector split(const std::string &text, char separator);
   bool isStartOfDoxyCommentChar(char c);
-  void addDoxyCommand(DoxygenParser::TokenList &tokList, const std::string &cmd);
+  bool addDoxyCommand(DoxygenParser::TokenList &tokList, const std::string &cmd);
 
 public:
   DoxygenParser(bool noisy = false);

Source/DoxygenTranslator/src/JavaDocConverter.cpp

@@ -147,6 +147,10 @@ void JavaDocConverter::fillStaticTables() {
   tagHandlers["plainstd::string"] = make_pair(&JavaDocConverter::handlePlainString, "");
   tagHandlers["plainstd::endl"] = make_pair(&JavaDocConverter::handleNewLine, "");
   tagHandlers["n"] = make_pair(&JavaDocConverter::handleNewLine, "");
+
+  // HTML tags
+  tagHandlers["<ul"] = make_pair(&JavaDocConverter::handleDoxyHtmlTag, "<ul");
+
 }
 
 
@@ -268,6 +272,8 @@ void JavaDocConverter::translateEntity(DoxygenEntity &tag,
 
   if (it != tagHandlers.end()) {
     (this->*(it->second.first))(tag, translatedComment, it->second.second);
+  } else {
+      addError(WARN_DOXYGEN_COMMAND_ERROR, "Unknown doxygen or HTML tag: " + tag.typeOfEntity);
   }
 }
 
@@ -285,6 +291,17 @@ void JavaDocConverter::handleTagHtml(DoxygenEntity& tag, std::string& translated
 }
 
 
+void JavaDocConverter::handleDoxyHtmlTag(DoxygenEntity& tag, std::string& translatedComment, std::string &arg) {
+    std::string htmlTagArgs = tag.data;
+    if (htmlTagArgs == "/") {
+        // end html tag, for example "</ul>
+        translatedComment += "</" + arg.substr(1) + ">";
+    } else {
+        translatedComment += arg + htmlTagArgs + ">";
+    }
+}
+
+
 void JavaDocConverter::handleNewLine(DoxygenEntity&, std::string& translatedComment, std::string&) {
   translatedComment += "\n * ";
 }
@@ -704,3 +721,10 @@ String *JavaDocConverter::makeDocumentation(Node *node) {
   
   return NewString(javaDocString.c_str());
 }
+
+
+void JavaDocConverter::addError(int warningType,
+                                     const std::string &message) {
+  Swig_warning(warningType, "", 0,
+               "Doxygen parser warning: %s. \n", message.c_str());
+}

Source/DoxygenTranslator/src/JavaDocConverter.h

@@ -60,6 +60,12 @@ protected:
    * arg - html tag, with no braces
    */
   void handleTagHtml(DoxygenEntity &tag, std::string &translatedComment, std::string &arg);
+
+  /* Handles HTML tags recognized by Doxygen, like <A ...>, <ul>, <table>, ... */
+  void handleDoxyHtmlTag(DoxygenEntity& tag,
+                            std::string& translatedComment,
+                            std::string &arg);
+
   /*
    * Just prints new line
    */
@@ -128,6 +134,8 @@ private:
 
   bool paramExists(std::string param);
   std::string indentAndInsertAsterisks(const std::string &doc);
+
+  void addError(int warningType, const std::string &message);
 };
 
 #endif