diff --git a/forms.tmpl b/forms.tmpl index bccfc34..6f48892 100644 --- a/forms.tmpl +++ b/forms.tmpl @@ -122,6 +122,7 @@
# for row in posts: # inc(count) +
@@ -251,3 +252,97 @@ ${stats.totalThreads} threads  |  ${stats.totalPosts} posts  |  newest member: ${stats.newestMember.nick} #end proc +# +# +# +# +#proc genSearchResults(c: var TForumData, +# results: iterator: db_sqlite.TRow {.closure, tags: [FReadDB].}, +# count: var int): string = +# const threadId = 0 +# const threadName = 1 +# const postId = 2 +# const postHeader = 3 +# const postContent = 4 +# const userName = 5 +# const postCreation = 6 +# const postAuthor = 7 +# const userEmail = 8 +# const what = 9 +# result = "" +# count = 0 +# var whCount: array[bool, int] +
+
+
+ Search results for: ${xmlEncode(c.search.replace(""","\""))}. +
+
+
+
+# for row in results(): +# inc(count) +# let isThread = %what == "0" +# inc(whCount[isThread]) +# let postUrl = c.genThreadUrl(%postId,"",%threadId,"0") +# let threadUrl = c.genThreadUrl("","",%threadId) +# var headersDiffer = false +
+
+
+ #let profileUrl = c.req.makeUri("profile/", false) & xmlEncode(%userName) + + + #if c.userId == %postAuthor and c.currentPost.subject.len == 0: +
Edit post + #elif c.isAdmin and c.currentPost.subject.len == 0: +
Edit post + #end if +
+
+
+
+ #if %threadName != %postHeader and "Re: " & %threadName != %postHeader: + #headersDiffer = true +

+ + Thread: + ${%threadName} + +

+ #end if + #if not headersDiffer or %postHeader != "": +
+ + Post: + ${%postHeader} + +
+ #end if + #if not isThread: + #try: + ${(%postContent)} + #except EParseError: + # c.errorMsg = getCurrentExceptionMsg() + #end + #end if + ${xmlEncode(%postCreation)} +
+
+
+# end for +
+# if c.pageNum > 1: +
+ + +
+# end if +# if whCount[true] == ThreadsPerPage or whCount[false] == ThreadsPerPage: +
+ + +
+# end if +#end proc +# diff --git a/forum.nim b/forum.nim index 61f0d9a..1af1812 100644 --- a/forum.nim +++ b/forum.nim @@ -8,7 +8,7 @@ import os, strutils, times, md5, strtabs, cgi, math, db_sqlite, matchers, - rst, rstgen, captchas, scgi, jester, asyncdispatch, asyncnet, cache + rst, rstgen, captchas, scgi, jester, asyncdispatch, asyncnet, cache, sequtils from htmlgen import tr, th, td, span const @@ -17,13 +17,14 @@ const ThreadsPerPage = 15 PostsPerPage = 10 + MaxPagesFromCurrent = 8 noPageNums = ["/login", "/register", "/dologin", "/doregister", "/profile"] noHomeBtn = ["/", "/login", "/register", "/dologin", "/doregister", "/profile"] type TCrud = enum crCreate, crRead, crUpdate, crDelete - TSession = object of TObject + TSession = object of RootObj threadid: int postid: int userName, userPass, email: string @@ -42,6 +43,8 @@ type isThreadsList: bool pageNum: int totalPosts: int + search: string + noPagenumumNav: bool TStyledButton = tuple[text: string, link: string] @@ -61,7 +64,8 @@ type var db: TDbConn - docConfig: PStringTable + docConfig: StringTableRef + isFTSAvailable: bool proc init(c: var TForumData) = c.userPass = "" @@ -75,6 +79,8 @@ proc init(c: var TForumData) = c.loginErrorMsg = "" c.invalidField = "" c.currentPost = (subject: "", content: "") + + c.search = "" proc loggedIn(c: TForumData): bool = result = c.userName.len > 0 @@ -114,6 +120,8 @@ proc genThreadUrl(c: TForumData, postId = "", action = "", threadid = "", pageNu result.add("?action=" & action) if postId != "": result.add("&postid=" & postid) + elif postId != "": + result.add("/" & postId & "#" & postId) result = c.req.makeUri(result, absolute = false) proc FormSession(c: var TForumData, nextAction: string): string = @@ -153,7 +161,8 @@ proc getGravatarUrl(email: string, size = 80): string = "&d=identicon") proc genGravatar(email: string, size: int = 80): string = - result = "" % getGravatarUrl(email, size) + result = "" % + [$size, $size, getGravatarUrl(email, size)] proc randomSalt(): string = result = "" @@ -179,7 +188,7 @@ proc makeSalt(): string = ## Creates a salt using a cryptographically secure random number generator. try: result = devRandomSalt() - except EIO: + except IOError: result = randomSalt() proc makePassword(password, salt: string): string = @@ -354,6 +363,8 @@ template setPreviewData(c: expr) {.immediate, dirty.} = template writeToDb(c, cr, setPostId: expr) = let retID = insertID(db, crud(cr, "post", "author", "ip", "header", "content", "thread"), c.userId, c.req.ip, subject, content, $c.threadId, "") + discard tryExec(db, crud(cr, "post_fts", "id", "header", "content"), + retID.int, subject, content) if setPostId: c.postId = retID.int @@ -366,17 +377,21 @@ proc edit(c: var TForumData, postId: int): bool = checkOwnership(c, $postId) if not tryExec(db, crud(crDelete, "post"), $postId): return setError(c, "", "database error") + discard tryExec(db, crud(crDelete, "post_fts"), $postId) # delete corresponding thread: if execAffectedRows(db, sql"delete from thread where id not in (select thread from post)") > 0: # whole thread has been deleted, so: c.threadId = unselectedThread + discard tryExec(db, sql"delete from thread_fts where id not in (select thread from post)") result = true else: checkOwnership(c, $postId) retrPost(c) exec(db, crud(crUpdate, "post", "header", "content"), subject, content, $postId) + exec(db, crud(crUpdate, "post_fts", "header", "content"), + subject, content, $postId) result = true proc reply(c: var TForumData): bool = @@ -401,7 +416,11 @@ proc newThread(c: var TForumData): bool = else: c.threadID = tryInsertID(db, query, c.req.params["subject"]).int if c.threadID < 0: return setError(c, "subject", "Subject already exists") + discard tryExec(db, crud(crCreate, "thread_fts", "id", "name"), + c.threadID, c.req.params["subject"]) writeToDb(c, crCreate, false) + discard tryExec(db, sql"insert into post_fts(post_fts) values('optimize')") + discard tryExec(db, sql"insert into post_fts(thread_fts) values('optimize')") result = true proc login(c: var TForumData, name, pass: string): bool = @@ -468,7 +487,7 @@ proc getStats(c: var TForumData, simple: bool): TForumStats = sql"select id, name, admin, strftime('%s', lastOnline), strftime('%s', creation) from person" for row in fastRows(db, getUsersQuery): let secs = if row[3] == "": 0 else: row[3].parseint - let lastOnlineSeconds = getTime() - TTime(secs) + let lastOnlineSeconds = getTime() - Time(secs) if lastOnlineSeconds < (60 * 5): # 5 minutes result.activeUsers.add((row[1], row[0].parseInt, row[2].parseBool)) if row[4].parseInt > newestMemberCreation: @@ -511,12 +530,17 @@ proc genPagenumNav(c: var TForumData, stats: TForumStats): string = else: firstTag = htmlgen.a(href=firstUrl, "<<") prevTag = htmlgen.a(href=prevUrl, "<••") + prevTag.add(htmlgen.link(rel="previous", href=prevUrl)) result.add(firstTag) result.add(prevTag) # Numbers var pages = "" # Tags - for i in 1..totalPages: + # cutting numbers to the left and to the right tp MaxPagesFromCurrent + let firstToShow = max(1, c.pageNum - MaxPagesFromCurrent) + let lastToShow = min(totalPages, c.pageNum + MaxPagesFromCurrent) + if firstToShow > 1: pages.add(span("...")) + for i in firstToShow .. lastToShow: if i == c.pageNum: pages.add(span($(i))) else: @@ -527,6 +551,16 @@ proc genPagenumNav(c: var TForumData, stats: TForumStats): string = pageUrl = c.req.makeUri(firstUrl & "/" & $(i)) pages.add(htmlgen.a(href = pageUrl, $(i))) + if lastToShow < totalPages: pages.add(span("...")) + # a number input for quick jump to a page + pages.add(htmlgen.form( + action = c.req.makeUri(if c.isThreadsList: "/page" else: firstUrl), + class = "pagenumJump", + when false: htmlgen.input(`type` = "number", name = "page", value = $c.pageNum, max = $totalPages) + else: "" % + [$c.pageNum, $totalPages], + htmlgen.input(`type` = "submit", value = "►", title = "Go to the page", class = "jump") )) + # max attribute for inputs not supported in htmlgen result.add(pages) # Right @@ -538,6 +572,7 @@ proc genPagenumNav(c: var TForumData, stats: TForumStats): string = else: lastTag = htmlgen.a(href=lastUrl, ">>") nextTag = htmlgen.a(href=nextUrl, "••>") + nextTag.add(htmlgen.link(rel="next",href=nextUrl)) result.add(nextTag) result.add(lastTag) @@ -619,7 +654,7 @@ proc genProfile(c: var TForumData, ui: TUserInfo): string = ) ) result.add(htmlgen.`div`(id = "avatar", genGravatar(ui.email, 250))) - let t2 = if ui.lastOnline != -1: getGMTime(TTime(ui.lastOnline)) + let t2 = if ui.lastOnline != -1: getGMTime(Time(ui.lastOnline)) else: getGMTime(getTime()) result.add(htmlgen.`div`(id = "info", @@ -688,14 +723,19 @@ routes: createTFD() resp genPostsRSS(c), "application/atom+xml" - get "/t/@threadid/?@page?/?": + get "/t/@threadid/?@page?/?@postid?/?": createTFD() parseInt(@"threadid", c.threadId, -1..1000_000) - if @"page".len > 0: - parseInt(@"page", c.pageNum, 0..1000_000) - cond (c.pageNum > 0) if (@"postid").len > 0: parseInt(@"postid", c.postId, -1..1000_000) + if @"page".len > 0: + parseInt(@"page", c.pageNum, 0..1000_000) + # for direct links to posts (with no thread id passed); used in search results + if c.pageNum == 0 and c.postId > 0: + const sqlGetPostsUnder = sql"select count(*) from post where thread = ? and id < ?" + let postsUnder = db.getValue(sqlGetPostsUnder, c.threadId, c.postId).parseInt + c.pageNum = (postsUnder div PostsPerPage) + 1 + cond (c.pageNum > 0) var count = 0 var pSubject = getThreadTitle(c.threadid, c.pageNum) cond validThreadId(c) @@ -719,6 +759,7 @@ routes: let content = ||row[1] body = genFormPost(c, "doedit", "Edit", header, content, true) title = "Editing post" + else: discard resp c.genMain(body, title & " - Nimrod Forum") else: incrementViews(c) @@ -726,7 +767,7 @@ routes: cond count != 0 resp genMain(c, posts, pSubject & " - Nimrod Forum") - get "/page/@page/?": + get "/page/?@page?/?": createTFD() c.isThreadsList = true cond (@"page" != "") @@ -779,7 +820,8 @@ routes: body.add genPostPreview(c, @"subject", @"content", c.userName, $getGMTime(getTime())) body.add genFormPost(c, action, topText, reuseText, reuseText, isEdit) - resp genMain(c, body(), "Nimrod Forum - Error") + resp genMain(c, body(), "Nimrod Forum - " & + (if c.isPreview: "Preview" else: "Error")) post "/dologin": createTFD() @@ -828,7 +870,7 @@ routes: createTFD() readIDs() if edit(c, c.postId): - redirect(c.genThreadUrl()) + redirect(c.genThreadUrl(pageNum = "0", postId = $c.postId)) else: body = "" handleError("doedit", "Edit", true) @@ -843,11 +885,58 @@ routes: createTFD() resp genMain(c, rstToHtml(licenseRst), "Content license - Nimrod Forum") + post "/search/?@page?": + if not isFTSAvailable: + redirect(uri("/")) + createTFD() + c.isThreadsList = true + c.noPagenumumNav = true + var count = 0 + var q = @"q" + for i in 0 .. q.len-1: + if q[i].int < 32: q[i] = ' ' + elif q[i] == '\'': q[i] = '"' + c.search = q.replace("\"","""); + if @"page".len > 0: + parseInt(@"page", c.pageNum, 0..1000_000) + cond (c.pageNum > 0) + iterator searchResults(): db_sqlite.TRow {.closure, tags: [FReadDB].} = + const queryFT = "fts.sql".slurp.sql + for rowFT in fastRows(db, queryFT, + [q,q,$ThreadsPerPage,$c.pageNum,$ThreadsPerPage,q, + q,q,$ThreadsPerPage,$c.pageNum,$ThreadsPerPage,q]): + yield rowFT + resp genMain(c, genSearchResults(c, searchResults, count), + additionalHeaders = genRSSHeaders(c), showRssLinks = true) + + # tries first to read html, then to read rst, convert ot html, cache and return + template textPage(path: string): stmt = + createTFD() + #c.isThreadsList = true + var page = "" + if existsFile(path): + page = readFile(path) + else: + let basePath = + if path[path.high] == '/': path & "index" + elif path.endsWith(".html"): path[-5 .. -1] + else: path + if existsFile(basePath & ".html"): + page = readFile(basePath & ".html") + elif existsFile(basePath & ".rst"): + page = readFile(basePath & ".rst").rstToHtml + writeFile(basePath & ".html", page) + resp genMain(c, page) + get "/search-help": + textPage "static/search-help" + when isMainModule: docConfig = rstgen.defaultConfig() math.randomize() db = open(connection="nimforum.db", user="postgres", password="", database="nimforum") + isFTSAvailable = db.getAllRows(sql("SELECT name FROM sqlite_master WHERE " & + "type='table' AND name='post_fts'")).len == 1 var http = true if paramCount() > 0: if paramStr(1) == "scgi": diff --git a/fts.sql b/fts.sql new file mode 100644 index 0000000..ab49d31 --- /dev/null +++ b/fts.sql @@ -0,0 +1,74 @@ +-- selects just threads, +-- those where title doesn't coinside with some of its posts' titles +-- by now selects only the threads title (no post snippet) +SELECT + thread_id, + snippet(thread_fts, '', '', '...') AS thread, + 0 AS post_id, + '' AS header, + '' AS content, + person.name AS author, + cdate, + author_id, + person.email AS email, + 0 AS what + FROM ( + SELECT + thread_fts.id AS thread_id, + post.id AS post_id, + post.creation AS cdate, + MIN(post.creation) AS cdate, + post.author AS author_id + FROM thread_fts + JOIN post ON post.thread=thread_id + WHERE thread_fts MATCH ? + GROUP BY thread_id, post_id + HAVING thread_id NOT IN ( + SELECT thread + FROM post_fts JOIN post USING(id) + WHERE post_fts MATCH ? + ) + LIMIT ? OFFSET (? - 1) * ? + ) + JOIN thread_fts ON thread_fts.id=thread_id + JOIN person ON person.id=author_id + WHERE thread_fts MATCH ? +UNION +-- the main query, selects posts +SELECT + thread.id AS thread_id, + thread.name AS thread, + post.id AS post_id, + CASE what WHEN 1 + THEN snippet(post_fts, '', '', '...', what) + ELSE post_fts.header END AS header, + CASE what WHEN 2 + THEN snippet(post_fts, '', '', '...', what, -45) + ELSE SUBSTR(post_fts.content, 1, 200) END AS content, + person.name AS author, + cdate, + post.author AS author_id, + person.email AS email, + what + FROM post_fts JOIN ( + -- inner query, selects ids of matching posts, orders and limits them, + -- so snippets only for limited count of posts are created (in outer query) + SELECT id, post.creation AS cdate, thread, 1 AS what, post.author AS author + FROM post_fts JOIN post USING(id) + WHERE post_fts.header MATCH ? + GROUP BY post.header + HAVING SUBSTR(post.header,1,3)<>'Re:' + UNION + SELECT id, post.creation AS cdate, thread, 2 AS what, post.author AS author + FROM post_fts JOIN post USING(id) + WHERE post_fts.content MATCH ? + ORDER BY what, cdate DESC + LIMIT ? OFFSET (? - 1) * ? + ) AS post USING(id) + JOIN thread ON thread.id=thread + JOIN person ON person.id=author + WHERE post_fts MATCH ? +ORDER BY what ASC, cdate DESC +LIMIT 300 -- hardcoded limit just in case +; + diff --git a/main.tmpl b/main.tmpl index c44dea3..cd96182 100644 --- a/main.tmpl +++ b/main.tmpl @@ -59,9 +59,11 @@
+ #if not c.noPagenumumNav:
${genPagenumNav(c, stats)}
+ #end if #elif hasReplyBtn(c):
@@ -82,8 +84,19 @@