Merge branch 'new_async' of https://github.com/Leu-Gim/nimforum into Leu-Gim-new_async

Conflicts:
	forum.nim

Also fixes FTS table detection.

forms.tmpl

@@ -122,6 +122,7 @@
 <div id="talk-thread">
 #  for row in posts:
 #    inc(count)
+  <a name="${%postId}"></a>
   <div>
     <div class="author">
       <div>
@@ -251,3 +252,97 @@
     <b>${stats.totalThreads}</b> threads &nbsp;|&nbsp; <b>${stats.totalPosts}</b> posts &nbsp;|&nbsp;
     newest member: <a href="$profileUrl">${stats.newestMember.nick}</a>
 #end proc
+#
+#
+#
+#
+#proc genSearchResults(c: var TForumData,
+#                       results: iterator: db_sqlite.TRow {.closure, tags: [FReadDB].},
+#                       count: var int): string =
+#  const threadId = 0
+#  const threadName = 1
+#  const postId = 2
+#  const postHeader = 3
+#  const postContent = 4
+#  const userName = 5
+#  const postCreation = 6
+#  const postAuthor = 7
+#  const userEmail = 8
+#  const what = 9
+#  result = ""
+#  count = 0
+#  var whCount: array[bool, int]
+<div id="talk-head">
+  <div class="info-post">
+    <div>
+        Search results for: <i style="color: #332299">${xmlEncode(c.search.replace("&quot;","\""))}</i>.
+    </div>
+  </div>
+</div>
+<div id="talk-thread" class="searchResults">
+#  for row in results():
+#    inc(count)
+#    let isThread = %what == "0"
+#    inc(whCount[isThread])
+#    let postUrl = c.genThreadUrl(%postId,"",%threadId,"0")
+#    let threadUrl = c.genThreadUrl("","",%threadId)
+#    var headersDiffer = false
+  <div>
+    <div class="author">
+      <div>
+        #let profileUrl = c.req.makeUri("profile/", false) & xmlEncode(%userName)
+        <div><a href="$profileUrl">${genGravatar(%userEmail, 40)}</a></div>
+        <div style="padding: 8px 0"><a href="$profileUrl">${xmlEncode(%userName)}</a></div>
+        #if c.userId == %postAuthor and c.currentPost.subject.len == 0:
+          <hr/><a href="${c.genThreadUrl(%postId, "edit", %threadId)}">Edit post</a>
+        #elif c.isAdmin and c.currentPost.subject.len == 0:
+          <hr/><a style="color: red;" href="${c.genThreadUrl(%postId, "edit", %threadId)}">Edit post</a>
+        #end if
+      </div>
+    </div>
+    <div class="topic">
+      <div>
+        #if %threadName != %postHeader and "Re: " & %threadName != %postHeader:
+        #headersDiffer = true
+        <h4>
+          <a href="${threadUrl}">
+            <span class="titleHeader">Thread:</span>
+            <span style="font-size: 1.2em">${%threadName}</span>
+          </a>
+        </h4>
+        #end if
+        #if not headersDiffer or %postHeader != "":
+        <h5>
+          <a href="${postUrl}">
+            <span class="titleHeader">Post:</span>
+            <span>${%postHeader}</span>
+          </a>
+        </h5>
+        #end if
+        #if not isThread:
+          #try:
+            ${(%postContent)}
+          #except EParseError:
+          #  c.errorMsg = getCurrentExceptionMsg()
+          #end
+        #end if
+        <span class="date">${xmlEncode(%postCreation)}</span>
+      </div>
+    </div>
+  </div>
+#  end for
+</div>
+#  if c.pageNum > 1:
+<form action="/search/${$(c.pageNum-1)}" method="post" class="searchNav">
+  <input type="hidden" name="q" value="${c.search}">
+  <input type="submit" value="Previous ${ThreadsPerPage} results">
+</form>
+#  end if
+#  if whCount[true] == ThreadsPerPage or whCount[false] == ThreadsPerPage:
+<form action="/search/${$(c.pageNum+1)}" method="post" class="searchNav">
+  <input type="hidden" name="q" value="${c.search}">
+  <input type="submit" value="Next ${ThreadsPerPage} results (if any)">
+</form>
+#  end if
+#end proc
+#

forum.nim

@@ -8,7 +8,7 @@
 
 import
   os, strutils, times, md5, strtabs, cgi, math, db_sqlite, matchers,
-  rst, rstgen, captchas, scgi, jester, asyncdispatch, asyncnet, cache
+  rst, rstgen, captchas, scgi, jester, asyncdispatch, asyncnet, cache, sequtils
 from htmlgen import tr, th, td, span
 
 const
@@ -17,13 +17,14 @@ const
 
   ThreadsPerPage = 15
   PostsPerPage = 10
+  MaxPagesFromCurrent = 8
   noPageNums = ["/login", "/register", "/dologin", "/doregister", "/profile"]
   noHomeBtn = ["/", "/login", "/register", "/dologin", "/doregister", "/profile"]
 
 type
   TCrud = enum crCreate, crRead, crUpdate, crDelete
 
-  TSession = object of TObject
+  TSession = object of RootObj
     threadid: int
     postid: int
     userName, userPass, email: string
@@ -42,6 +43,8 @@ type
     isThreadsList: bool
     pageNum: int
     totalPosts: int
+    search: string
+    noPagenumumNav: bool
   
   TStyledButton = tuple[text: string, link: string]
 
@@ -61,7 +64,8 @@ type
 
 var
   db: TDbConn
-  docConfig: PStringTable
+  docConfig: StringTableRef
+  isFTSAvailable: bool
   
 proc init(c: var TForumData) = 
   c.userPass = ""
@@ -75,6 +79,8 @@ proc init(c: var TForumData) =
   c.loginErrorMsg = ""
   c.invalidField = ""
   c.currentPost = (subject: "", content: "")
+  
+  c.search = ""
 
 proc loggedIn(c: TForumData): bool = 
   result = c.userName.len > 0
@@ -114,6 +120,8 @@ proc genThreadUrl(c: TForumData, postId = "", action = "", threadid = "", pageNu
     result.add("?action=" & action)
     if postId != "":
       result.add("&postid=" & postid)
+  elif postId != "":
+    result.add("/" & postId & "#" & postId)
   result = c.req.makeUri(result, absolute = false)
 
 proc FormSession(c: var TForumData, nextAction: string): string =
@@ -153,7 +161,8 @@ proc getGravatarUrl(email: string, size = 80): string =
      "&d=identicon")
 
 proc genGravatar(email: string, size: int = 80): string =
-  result = "<img src=\"$1\" />" % getGravatarUrl(email, size)
+  result = "<img width=\"$1\" height=\"$2\" src=\"$3\" />" % 
+            [$size, $size, getGravatarUrl(email, size)]
 
 proc randomSalt(): string =
   result = ""
@@ -179,7 +188,7 @@ proc makeSalt(): string =
   ## Creates a salt using a cryptographically secure random number generator.
   try:
     result = devRandomSalt()
-  except EIO:
+  except IOError:
     result = randomSalt()
 
 proc makePassword(password, salt: string): string =
@@ -354,6 +363,8 @@ template setPreviewData(c: expr) {.immediate, dirty.} =
 template writeToDb(c, cr, setPostId: expr) =
   let retID = insertID(db, crud(cr, "post", "author", "ip", "header", "content", "thread"),
        c.userId, c.req.ip, subject, content, $c.threadId, "")
+  discard tryExec(db, crud(cr, "post_fts", "id", "header", "content"),
+       retID.int, subject, content)
   if setPostId:
     c.postId = retID.int
 
@@ -366,17 +377,21 @@ proc edit(c: var TForumData, postId: int): bool =
     checkOwnership(c, $postId)
     if not tryExec(db, crud(crDelete, "post"), $postId):
       return setError(c, "", "database error")
+    discard tryExec(db, crud(crDelete, "post_fts"), $postId)
     # delete corresponding thread:
     if execAffectedRows(db,
         sql"delete from thread where id not in (select thread from post)") > 0:
       # whole thread has been deleted, so:
       c.threadId = unselectedThread
+      discard tryExec(db, sql"delete from thread_fts where id not in (select thread from post)")
     result = true
   else:
     checkOwnership(c, $postId)
     retrPost(c)
     exec(db, crud(crUpdate, "post", "header", "content"),
          subject, content, $postId)
+    exec(db, crud(crUpdate, "post_fts", "header", "content"),
+         subject, content, $postId)
     result = true
   
 proc reply(c: var TForumData): bool = 
@@ -401,7 +416,11 @@ proc newThread(c: var TForumData): bool =
   else:
     c.threadID = tryInsertID(db, query, c.req.params["subject"]).int
     if c.threadID < 0: return setError(c, "subject", "Subject already exists")
+    discard tryExec(db, crud(crCreate, "thread_fts", "id", "name"),
+                        c.threadID, c.req.params["subject"])
     writeToDb(c, crCreate, false)
+    discard tryExec(db, sql"insert into post_fts(post_fts) values('optimize')")
+    discard tryExec(db, sql"insert into post_fts(thread_fts) values('optimize')")
     result = true
 
 proc login(c: var TForumData, name, pass: string): bool = 
@@ -468,7 +487,7 @@ proc getStats(c: var TForumData, simple: bool): TForumStats =
       sql"select id, name, admin, strftime('%s', lastOnline), strftime('%s', creation) from person"
     for row in fastRows(db, getUsersQuery):
       let secs = if row[3] == "": 0 else: row[3].parseint
-      let lastOnlineSeconds = getTime() - TTime(secs)
+      let lastOnlineSeconds = getTime() - Time(secs)
       if lastOnlineSeconds < (60 * 5): # 5 minutes
         result.activeUsers.add((row[1], row[0].parseInt, row[2].parseBool))
       if row[4].parseInt > newestMemberCreation:
@@ -511,12 +530,17 @@ proc genPagenumNav(c: var TForumData, stats: TForumStats): string =
   else:
     firstTag = htmlgen.a(href=firstUrl, "<<")
     prevTag = htmlgen.a(href=prevUrl, "<••")
+    prevTag.add(htmlgen.link(rel="previous", href=prevUrl))
   result.add(firstTag)
   result.add(prevTag)
   
   # Numbers
   var pages = "" # Tags
-  for i in 1..totalPages:
+  # cutting numbers to the left and to the right tp MaxPagesFromCurrent
+  let firstToShow = max(1, c.pageNum - MaxPagesFromCurrent)
+  let lastToShow  = min(totalPages, c.pageNum + MaxPagesFromCurrent)
+  if firstToShow > 1: pages.add(span("..."))
+  for i in firstToShow .. lastToShow:
     if i == c.pageNum:
       pages.add(span($(i)))
     else:
@@ -527,6 +551,16 @@ proc genPagenumNav(c: var TForumData, stats: TForumStats): string =
         pageUrl = c.req.makeUri(firstUrl & "/" & $(i))
       
       pages.add(htmlgen.a(href = pageUrl, $(i)))
+  if lastToShow < totalPages: pages.add(span("..."))
+  # a number input for quick jump to a page
+  pages.add(htmlgen.form(
+    action = c.req.makeUri(if c.isThreadsList: "/page" else: firstUrl),
+    class  = "pagenumJump",
+    when false: htmlgen.input(`type` = "number", name = "page", value = $c.pageNum, max = $totalPages)
+    else: "<input type=\"number\" name=\"page\" value=\"$1\" min=\"1\" max=\"$2\" />" %
+            [$c.pageNum, $totalPages],
+    htmlgen.input(`type` = "submit", value = "&#9658;", title = "Go to the page", class = "jump") ))
+  # max attribute for inputs not supported in htmlgen
   result.add(pages)
 
   # Right
@@ -538,6 +572,7 @@ proc genPagenumNav(c: var TForumData, stats: TForumStats): string =
   else:
     lastTag = htmlgen.a(href=lastUrl, ">>")
     nextTag = htmlgen.a(href=nextUrl, "••>")
+    nextTag.add(htmlgen.link(rel="next",href=nextUrl))
   result.add(nextTag)
   result.add(lastTag)
 
@@ -619,7 +654,7 @@ proc genProfile(c: var TForumData, ui: TUserInfo): string =
     )
   )
   result.add(htmlgen.`div`(id = "avatar", genGravatar(ui.email, 250)))
-  let t2 = if ui.lastOnline != -1: getGMTime(TTime(ui.lastOnline)) 
+  let t2 = if ui.lastOnline != -1: getGMTime(Time(ui.lastOnline)) 
            else: getGMTime(getTime())
   
   result.add(htmlgen.`div`(id = "info", 
@@ -688,14 +723,19 @@ routes:
     createTFD()
     resp genPostsRSS(c), "application/atom+xml"
 
-  get "/t/@threadid/?@page?/?":
+  get "/t/@threadid/?@page?/?@postid?/?":
     createTFD()
     parseInt(@"threadid", c.threadId, -1..1000_000)
-    if @"page".len > 0:
-      parseInt(@"page", c.pageNum, 0..1000_000)
-      cond (c.pageNum > 0)
     if (@"postid").len > 0:
       parseInt(@"postid", c.postId, -1..1000_000)
+    if @"page".len > 0:
+      parseInt(@"page", c.pageNum, 0..1000_000)
+    # for direct links to posts (with no thread id passed); used in search results
+    if c.pageNum == 0 and c.postId > 0:
+      const sqlGetPostsUnder = sql"select count(*) from post where thread = ? and id < ?"
+      let postsUnder = db.getValue(sqlGetPostsUnder, c.threadId, c.postId).parseInt
+      c.pageNum = (postsUnder div PostsPerPage) + 1
+    cond (c.pageNum > 0)
     var count = 0
     var pSubject = getThreadTitle(c.threadid, c.pageNum)
     cond validThreadId(c)
@@ -719,6 +759,7 @@ routes:
         let content = ||row[1]
         body = genFormPost(c, "doedit", "Edit", header, content, true)
         title = "Editing post"
+      else: discard
       resp c.genMain(body, title & " - Nimrod Forum")
     else:
       incrementViews(c)
@@ -726,7 +767,7 @@ routes:
       cond count != 0
       resp genMain(c, posts, pSubject & " - Nimrod Forum")
 
-  get "/page/@page/?":
+  get "/page/?@page?/?":
     createTFD()
     c.isThreadsList = true
     cond (@"page" != "")
@@ -779,7 +820,8 @@ routes:
       body.add genPostPreview(c, @"subject", @"content", 
                               c.userName, $getGMTime(getTime()))
     body.add genFormPost(c, action, topText, reuseText, reuseText, isEdit)
-    resp genMain(c, body(), "Nimrod Forum - Error")
+    resp genMain(c, body(), "Nimrod Forum - " & 
+                            (if c.isPreview: "Preview" else: "Error"))
 
   post "/dologin":
     createTFD()
@@ -828,7 +870,7 @@ routes:
     createTFD()
     readIDs()
     if edit(c, c.postId):
-      redirect(c.genThreadUrl())
+      redirect(c.genThreadUrl(pageNum = "0", postId = $c.postId))
     else:
       body = ""
       handleError("doedit", "Edit", true)
@@ -843,11 +885,58 @@ routes:
     createTFD()
     resp genMain(c, rstToHtml(licenseRst), "Content license - Nimrod Forum")
 
+  post "/search/?@page?":
+    if not isFTSAvailable:
+      redirect(uri("/"))
+    createTFD()
+    c.isThreadsList  = true
+    c.noPagenumumNav = true
+    var count = 0
+    var q = @"q"
+    for i in 0 .. q.len-1:
+      if   q[i].int < 32: q[i] = ' '
+      elif q[i] == '\'':  q[i] = '"'
+    c.search = q.replace("\"","&quot;");
+    if @"page".len > 0:
+      parseInt(@"page", c.pageNum, 0..1000_000)
+      cond (c.pageNum > 0)
+    iterator searchResults(): db_sqlite.TRow {.closure, tags: [FReadDB].} =
+      const queryFT = "fts.sql".slurp.sql
+      for rowFT in fastRows(db, queryFT,
+                    [q,q,$ThreadsPerPage,$c.pageNum,$ThreadsPerPage,q,
+                     q,q,$ThreadsPerPage,$c.pageNum,$ThreadsPerPage,q]):
+        yield rowFT
+    resp genMain(c, genSearchResults(c, searchResults, count),
+                 additionalHeaders = genRSSHeaders(c), showRssLinks = true)
+
+  # tries first to read html, then to read rst, convert ot html, cache and return
+  template textPage(path: string): stmt =
+    createTFD()
+    #c.isThreadsList = true
+    var page = ""
+    if existsFile(path):
+      page = readFile(path)
+    else:
+      let basePath = 
+        if path[path.high] == '/': path & "index"
+        elif path.endsWith(".html"): path[-5 .. -1]
+        else: path
+      if existsFile(basePath & ".html"):
+        page = readFile(basePath & ".html")
+      elif existsFile(basePath & ".rst"):
+        page = readFile(basePath & ".rst").rstToHtml
+        writeFile(basePath & ".html", page)
+    resp genMain(c, page)
+  get "/search-help":
+    textPage "static/search-help"
+
 when isMainModule:
   docConfig = rstgen.defaultConfig()
   math.randomize()
   db = open(connection="nimforum.db", user="postgres", password="", 
               database="nimforum")
+  isFTSAvailable = db.getAllRows(sql("SELECT name FROM sqlite_master WHERE " &
+      "type='table' AND name='post_fts'")).len == 1
   var http = true
   if paramCount() > 0:
     if paramStr(1) == "scgi":

fts.sql

@@ -0,0 +1,74 @@
+-- selects just threads,
+-- those where title doesn't coinside with some of its posts' titles
+-- by now selects only the threads title (no post snippet)
+SELECT
+        thread_id,
+        snippet(thread_fts, '<b>', '</b>', '<b>...</b>') AS thread,
+        0 AS post_id,
+        '' AS header,
+        '' AS content,
+        person.name AS author,
+        cdate,
+        author_id,
+        person.email AS email,
+        0 AS what
+    FROM (
+        SELECT
+                thread_fts.id AS thread_id,
+                post.id AS post_id,
+                post.creation AS cdate,
+                MIN(post.creation) AS cdate,
+                post.author AS author_id
+            FROM thread_fts
+            JOIN post ON post.thread=thread_id
+            WHERE thread_fts MATCH ?
+            GROUP BY thread_id, post_id
+            HAVING thread_id NOT IN (
+                SELECT thread
+                    FROM post_fts JOIN post USING(id)
+                    WHERE post_fts MATCH ?
+            )
+            LIMIT ? OFFSET (? - 1) * ?
+    )
+        JOIN thread_fts ON thread_fts.id=thread_id
+        JOIN person ON person.id=author_id
+    WHERE thread_fts MATCH ?
+UNION
+-- the main query, selects posts
+SELECT
+        thread.id AS thread_id,
+        thread.name AS thread,
+        post.id AS post_id,
+        CASE what WHEN 1
+            THEN snippet(post_fts, '<b>', '</b>', '...', what)
+            ELSE post_fts.header END AS header,
+        CASE what WHEN 2
+            THEN snippet(post_fts, '<b>', '</b>', '...', what, -45)
+            ELSE SUBSTR(post_fts.content, 1, 200) END AS content,
+        person.name AS author,
+        cdate,
+        post.author AS author_id,
+        person.email AS email,
+        what
+    FROM post_fts JOIN (
+    -- inner query, selects ids of matching posts, orders and limits them,
+    -- so snippets only for limited count of posts are created (in outer query)
+        SELECT id, post.creation AS cdate, thread, 1 AS what, post.author AS author
+            FROM post_fts JOIN post USING(id)
+            WHERE post_fts.header MATCH ?
+            GROUP BY post.header
+            HAVING SUBSTR(post.header,1,3)<>'Re:'
+        UNION
+        SELECT id, post.creation AS cdate, thread, 2 AS what, post.author AS author
+            FROM post_fts JOIN post USING(id)
+            WHERE post_fts.content MATCH ?
+        ORDER BY what, cdate DESC
+        LIMIT ? OFFSET (? - 1) * ?
+    ) AS post USING(id)
+        JOIN thread ON thread.id=thread
+        JOIN person ON person.id=author
+    WHERE post_fts MATCH ?
+ORDER BY what ASC, cdate DESC
+LIMIT 300 -- hardcoded limit just in case
+;
+

main.tmpl

@@ -59,9 +59,11 @@
               </div>
             </div>
           </div>
+          #if not c.noPagenumumNav:
           <div id="talk-nav">
             ${genPagenumNav(c, stats)}
           </div>
+          #end if
         #elif hasReplyBtn(c):
           <div id="talk-info">
             <div class="info-post">
@@ -82,8 +84,19 @@
       <div id="sidebar">
         <div class="title">Search</div>
         <div class="content">
-          <input type="text" />
-          <a href="#" class="button search">Search</a>
+          #if isFTSAvailable:
+            <a href="/search-help" target="_blank" class="searchHelp">?</a>
+            <form method="post" action="/search" class="searchForm">
+              <input type="text" name="q" maxlength="255" value="${c.search}" title="Search this forum" />
+              <input type="submit" value="Search" class="button search">
+            </form>
+          #else:
+            <form method="get" action="http://www.google.com/search" target="_blank" class="searchForm">
+              <input type="text" name="q" maxlength="255" value="${c.search}" title="Search this forum" />
+              <input type="submit" value="Search" class="button search">
+              <input type="hidden"  name="sitesearch" value="http://forum.nimrod-lang.org" />
+            </form>
+          #end if
         </div>
         #if c.loggedIn:
           <div class="title">Your account</div>

public/css/style.css

@@ -501,3 +501,84 @@ hr
 {
   display: none;
 }
+
+
+/* highlighting current post */
+
+.post.current {
+	background-color: #fefff3;
+}
+
+/* full-text search */
+
+.searchResults h4 b,
+.searchResults h5 b {
+  border-bottom: 1px dotted #ffffff;
+}
+.titleHeader {
+  margin-right: 1em;
+  color: #ddd;
+}
+
+.searchForm {
+  margin-top: 0px;
+  margin-right: 1em;
+  margin-bottom: 0px;
+  margin-left: 1em;
+}
+
+.searchHelp { 
+  color: #000000 !important;
+  float: right;
+  font-size: 11px;
+  left: -17px;
+  top: 3px;
+  position: relative;
+  text-decoration: none;
+  text-shadow: #FFFF00 1px 1px 2px;
+  cursor: help;
+}
+
+#talk-thread.searchResults > div > div > div {
+  margin: 15px 8px;
+}
+
+form.searchNav {
+  display: inline;
+  border: none !important;
+  background: transparent !important;
+}
+
+.searchNav input {
+  background: #858C97;
+  color: #000000;
+  border: 1px solid #333;
+}
+
+.clear {
+  clear: both;
+  height: 1px;
+}
+
+/* page numbers form */
+
+.pagenumJump {
+  display: inline;
+  border-width: 0px !important;
+}
+
+.pagenumJump input {
+  margin: 0 .2em;
+  width: 3.2em;
+  background: #d0d6e3;
+  padding: 1px 2px;
+  border-width: 0px;
+}
+
+.pagenumJump .jump {
+  border-width: 0px;
+  margin: 0;
+  background: transparent;
+  width: auto;
+  cursor: pointer;
+}

static/search-help.rst

@@ -0,0 +1,35 @@
+Full-text search for Nim forum
+==============================
+
+Syntax (using *SQLite* dll compiled without *Enhanced Query Syntax* support):
+-----------------------------------------------------------------------------
+
+- Only alphanumeric characters are searched.
+- Only full words and words beginnings (e.g. ``Nim*`` for both ``Nimrod`` and ``Nim``) are searched
+- All words are joined with implicit **AND** operator; there's no explicit one
+- There's explicit **OR** operator (upper-case) and it has higher priority
+- Words can be prepended with **-** to be excluded from search
+- No parentheses support
+- Quotes for phrases search, e.g. ``"programming language"``
+- Distances between words/phrases can be specified putting ``NEAR`` or ``NEAR/some_number`` between them
+
+Syntax - differences in *Enhanced Query Syntax* (should be enabled in *SQLite* dll):
+------------------------------------------------------------------------------------
+
+- **AND** and **NOT** logical operators available
+- Precedence of operators is, from highest to lowest: **NOT**, **AND**, **OR**
+- Parentheses for grouping are supported
+
+Where search is performed:
+--------------------------
+
+- **Threads' titles** - these results are outputed first
+- **Posts' titles** - middle precedence
+- **Posts' contents** - the latest
+
+How results are shown:
+----------------------
+
+- All results are ordered by date (posts' edits don't affect)
+- Matched tokens in text are marked (bold or dotted underline)
+- Threads title is the link to the thread and posts title is the link to the post