Add changes required by Nimble lock file support (#12104)

Implemented support for Nimble local cache with package directories with
a checksum of the package at the end of their names. Now the compiler
supports package paths in the form:

 * /path_to_nimble_cache_dir/pkgs/package_name-1.2.3-
FEBADEAEA2345E777F0F6F8433F7F0A52EDD5D1B

 * /path_to_nimble_cache_dir/pkgs/package_name-#head-
042D4BE2B90ED0672E717D71850ABDB0A2D19CD2

 * /path_to_nimble_cache_dir/pkgs/package_name-#branch-name-
DBC1F902CB79946E990E38AF51F0BAD36ACFABD9

Related to nim-lang/nimble#127

compiler/modulepaths.nim

@@ -21,7 +21,7 @@ when false:
     for k, p in os.walkDir(dir, relative=true):
       if k == pcDir and p.len > pkg.len+1 and
           p[pkg.len] == '-' and p.startsWith(pkg):
-        let (_, a) = getPathVersion(p)
+        let (_, a, _) = getPathVersionChecksum(p)
         if bestv.len == 0 or bestv < a:
           bestv = a
           best = dir / p

compiler/nimblecmd.nim

@@ -9,8 +9,8 @@
 
 ## Implements some helper procs for Nimble (Nim's package manager) support.
 
-import parseutils, strutils, strtabs, os, options, msgs, sequtils,
+import parseutils, strutils, os, options, msgs, sequtils, lineinfos, pathutils,
-  lineinfos, pathutils
+  std/sha1, tables
 
 proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if not conf.searchPaths.contains(path):
@@ -18,6 +18,7 @@ proc addPath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
 
 type
   Version* = distinct string
+  PackageInfo = Table[string, tuple[version, checksum: string]]
 
 proc `$`*(ver: Version): string {.borrow.}
 
@@ -62,43 +63,64 @@ proc `<`*(ver: Version, ver2: Version): bool =
     else:
       return false
 
-proc getPathVersion*(p: string): tuple[name, version: string] =
+proc getPathVersionChecksum*(p: string): tuple[name, version, checksum: string] =
-  ## Splits path ``p`` in the format ``/home/user/.nimble/pkgs/package-0.1``
+  ## Splits path ``p`` in the format
-  ## into ``(/home/user/.nimble/pkgs/package, 0.1)``
+  ## ``/home/user/.nimble/pkgs/package-0.1-febadeaea2345e777f0f6f8433f7f0a52edd5d1b`` into
-  result.name = ""
+  ## ``("/home/user/.nimble/pkgs/package", "0.1", "febadeaea2345e777f0f6f8433f7f0a52edd5d1b")``
-  result.version = ""
 
-  const specialSeparator = "-#"
+  const checksumSeparator = '-'
-  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
+  const versionSeparator = '-'
-  var sepIdx = p.find(specialSeparator, start = last)
+  const specialVersionSepartator = "-#"
-  if sepIdx == -1:
+  const separatorNotFound = -1
-    sepIdx = p.rfind('-', start = last)
 
-  if sepIdx == -1:
+  var checksumSeparatorIndex = p.rfind(checksumSeparator)
-    result.name = p
+  if checksumSeparatorIndex != separatorNotFound:
-    return
+    result.checksum = p.substr(checksumSeparatorIndex + 1)
+    if not result.checksum.isValidSha1Hash():
+      result.checksum = ""
+      checksumSeparatorIndex = p.len()
+  else:
+    checksumSeparatorIndex = p.len()
 
-  for i in sepIdx..<p.len:
+  var versionSeparatorIndex = p.rfind(
-    if p[i] in {DirSep, AltSep}:
+    specialVersionSepartator, 0, checksumSeparatorIndex - 1)
-      result.name = p
+  if versionSeparatorIndex != separatorNotFound:
-      return
+    result.version = p.substr(
+      versionSeparatorIndex + 1, checksumSeparatorIndex - 1)
+  else:
+    versionSeparatorIndex = p.rfind(
+      versionSeparator, 0, checksumSeparatorIndex - 1)
+    if versionSeparatorIndex != separatorNotFound:
+      result.version = p.substr(
+        versionSeparatorIndex + 1, checksumSeparatorIndex - 1)
+    else:
+      versionSeparatorIndex = checksumSeparatorIndex
 
-  result.name = p[0..sepIdx - 1]
+  result.name = p[0..<versionSeparatorIndex]
-  result.version = p.substr(sepIdx + 1)
 
-proc addPackage(conf: ConfigRef; packages: StringTableRef, p: string; info: TLineInfo) =
+proc addPackage*(conf: ConfigRef; packages: var PackageInfo, p: string;
-  let (name, ver) = getPathVersion(p)
+                 info: TLineInfo) =
+  let (name, ver, checksum) = getPathVersionChecksum(p)
   if isValidVersion(ver):
     let version = newVersion(ver)
-    if packages.getOrDefault(name).newVersion < version or
+    if packages.getOrDefault(name).version.newVersion < version or
       (not packages.hasKey(name)):
-      packages[name] = $version
+      if checksum.isValidSha1Hash():
+        packages[name] = ($version, checksum)
+      else:
+        packages[name] = ($version, "")
   else:
     localError(conf, info, "invalid package name: " & p)
 
-iterator chosen(packages: StringTableRef): string =
+iterator chosen(packages: PackageInfo): string =
   for key, val in pairs(packages):
-    let res = if val.len == 0: key else: key & '-' & val
+    var res = key
+    if val.version.len != 0:
+      res &= '-'
+      res &= val.version
+    if val.checksum.len != 0:
+      res &= '-'
+      res &= val.checksum
     yield res
 
 proc addNimblePath(conf: ConfigRef; p: string, info: TLineInfo) =
@@ -118,7 +140,7 @@ proc addNimblePath(conf: ConfigRef; p: string, info: TLineInfo) =
     conf.lazyPaths.insert(AbsoluteDir path, 0)
 
 proc addPathRec(conf: ConfigRef; dir: string, info: TLineInfo) =
-  var packages = newStringTable(modeStyleInsensitive)
+  var packages: PackageInfo
   var pos = dir.len-1
   if dir[pos] in {DirSep, AltSep}: inc(pos)
   for k,p in os.walkDir(dir):

lib/std/sha1.nim

@@ -275,3 +275,7 @@ proc `==`*(a, b: SecureHash): bool =
 
   # Not a constant-time comparison, but that's acceptable in this context
   Sha1Digest(a) == Sha1Digest(b)
+
+proc isValidSha1Hash*(s: string): bool =
+  ## Checks if a string is a valid sha1 hash sum.
+  s.len == 40 and allCharsInSet(s, HexDigits)

tests/compiler/tnimblecmd.nim

@@ -1,26 +1,75 @@
-include compiler/[nimblecmd]
+include compiler/[nimblecmd], sets
 
 proc v(s: string): Version = s.newVersion
-# #head is special in the sense that it's assumed to always be newest.
-doAssert v"1.0" < v"#head"
-doAssert v"1.0" < v"1.1"
-doAssert v"1.0.1" < v"1.1"
-doAssert v"1" < v"1.1"
-doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
-doAssert v"#a111" < v"#head"
 
-let conf = newConfigRef()
+proc testVersionsComparison =
-var rr = newStringTable()
+  # #head is special in the sense that it's assumed to always be newest.
-addPackage conf, rr, "irc-#a111", unknownLineInfo
+  doAssert v"1.0" < v"#head"
-addPackage conf, rr, "irc-#head", unknownLineInfo
+  doAssert v"1.0" < v"1.1"
-addPackage conf, rr, "irc-0.1.0", unknownLineInfo
+  doAssert v"1.0.1" < v"1.1"
-#addPackage conf, rr, "irc", unknownLineInfo
+  doAssert v"1" < v"1.1"
-#addPackage conf, rr, "another", unknownLineInfo
+  doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
-addPackage conf, rr, "another-0.1", unknownLineInfo
+  doAssert v"#a111" < v"#head"
 
-addPackage conf, rr, "ab-0.1.3", unknownLineInfo
+proc testAddPackageWithoutChecksum =
-addPackage conf, rr, "ab-0.1", unknownLineInfo
+  ## For backward compatibility it is not required all packages to have a
-addPackage conf, rr, "justone-1.0", unknownLineInfo
+  ## sha1 checksum at the end of the name of the Nimble cache directory.
+  ## This way a new compiler will be able to work with an older Nimble.
 
-doAssert toSeq(rr.chosen) ==
+  let conf = newConfigRef()
-  @["irc-#head", "ab-0.1.3", "justone-1.0", "another-0.1"]
+  var rr: PackageInfo
+
+  addPackage conf, rr, "irc-#a111", unknownLineInfo
+  addPackage conf, rr, "irc-#head", unknownLineInfo
+  addPackage conf, rr, "irc-0.1.0", unknownLineInfo
+
+  addPackage conf, rr, "another-0.1", unknownLineInfo
+
+  addPackage conf, rr, "ab-0.1.3", unknownLineInfo
+  addPackage conf, rr, "ab-0.1", unknownLineInfo
+  addPackage conf, rr, "justone-1.0", unknownLineInfo
+
+  doAssert toSeq(rr.chosen).toHashSet ==
+    ["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"].toHashSet
+
+proc testAddPackageWithChecksum =
+  let conf = newConfigRef()
+  var rr: PackageInfo
+
+  # in the case of packages with the same version, but different checksums for
+  # now the first one will be chosen
+
+  addPackage conf, rr, "irc-#a111-DBC1F902CB79946E990E38AF51F0BAD36ACFABD9",
+             unknownLineInfo
+  addPackage conf, rr, "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD1",
+             unknownLineInfo
+  addPackage conf, rr, "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD2",
+             unknownLineInfo
+  addPackage conf, rr, "irc-0.1.0-6EE6DE936B32E82C7DBE526DA3463574F6568FAF",
+             unknownLineInfo
+
+  addPackage conf, rr, "another-0.1", unknownLineInfo
+  addPackage conf, rr, "another-0.1-F07EE6040579F0590608A8FD34F5F2D91D859340",
+             unknownLineInfo
+
+  addPackage conf, rr, "ab-0.1.3-34BC3B72CE46CF5A496D1121CFEA7369385E9EA2",
+             unknownLineInfo
+  addPackage conf, rr, "ab-0.1.3-24BC3B72CE46CF5A496D1121CFEA7369385E9EA2",
+             unknownLineInfo
+  addPackage conf, rr, "ab-0.1-A3CFFABDC4759F7779D541F5E031AED17169390A",
+             unknownLineInfo
+
+  # lower case hex digits is also a valid sha1 checksum
+  addPackage conf, rr, "justone-1.0-f07ee6040579f0590608a8fd34f5f2d91d859340",
+             unknownLineInfo
+
+  doAssert toSeq(rr.chosen).toHashSet == [
+    "irc-#head-042D4BE2B90ED0672E717D71850ABDB0A2D19CD1",
+    "another-0.1",
+    "ab-0.1.3-34BC3B72CE46CF5A496D1121CFEA7369385E9EA2",
+    "justone-1.0-f07ee6040579f0590608a8fd34f5f2d91d859340"
+    ].toHashSet
+
+testVersionsComparison()
+testAddPackageWithoutChecksum()
+testAddPackageWithChecksum()

tests/stdlib/tsha1.nim

@@ -11,3 +11,13 @@ checkVector("", "da39a3ee5e6b4b0d3255bfef95601890afd80709")
 checkVector("abc", "a9993e364706816aba3e25717850c26c9cd0d89d")
 checkVector("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq",
             "84983e441c3bd26ebaae4aa1f95129e5e54670f1")
+
+proc testIsValidSha1Hash =
+  doAssert not isValidSha1Hash("")
+  doAssert not isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD11")
+  doAssert not isValidSha1hash("042G4BE2B90ED0672E717D71850ABDB0A2D19CD1")
+  doAssert isValidSha1Hash("042D4BE2B90ED0672E717D71850ABDB0A2D19CD1")
+  doAssert isValidSha1Hash("042d4be2b90ed0672e717d71850abdb0a2d19cd1")
+  doAssert isValidSha1Hash("042d4be2b90ed0672e717D71850ABDB0A2D19CD1")
+
+testIsValidSha1Hash()