# Copyright (C) Dominik Picheta. All rights reserved. # BSD License. Look at license.txt for more info. import parsecfg, json, sets, streams, strutils, parseutils, os, tables, sugar from sequtils import apply, map import version, tools, common, nimscriptwrapper, options, packageinfo, cli ## Contains procedures for parsing .nimble files. Moved here from ``packageinfo`` ## because it depends on ``nimscriptwrapper`` (``nimscriptwrapper`` also ## depends on other procedures in ``packageinfo``. type NimbleFile* = string ValidationError* = object of NimbleError warnInstalled*: bool # Determines whether to show a warning for installed pkgs warnAll*: bool const reservedNames = [ "CON", "PRN", "AUX", "NUL", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3", "LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9", ] proc newValidationError(msg: string, warnInstalled: bool, hint: string, warnAll: bool): ref ValidationError = result = newException(ValidationError, msg) result.warnInstalled = warnInstalled result.warnAll = warnAll result.hint = hint proc raiseNewValidationError(msg: string, warnInstalled: bool, hint: string = "", warnAll = false) = raise newValidationError(msg, warnInstalled, hint, warnAll) proc validatePackageName*(name: string) = ## Raises an error if specified package name contains invalid characters. ## ## A valid package name is one which is a valid nim module name. So only ## underscores, letters and numbers allowed. if name.len == 0: return if name[0] in {'0'..'9'}: raiseNewValidationError(name & "\"$1\" is an invalid package name: cannot begin with $2" % [name, $name[0]], true) var prevWasUnderscore = false for c in name: case c of '_': if prevWasUnderscore: raiseNewValidationError( "$1 is an invalid package name: cannot contain \"__\"" % name, true) prevWasUnderscore = true of AllChars - IdentChars: raiseNewValidationError( "$1 is an invalid package name: cannot contain '$2'" % [name, $c], true) else: prevWasUnderscore = false if name.endsWith("pkg"): raiseNewValidationError("\"$1\" is an invalid package name: cannot end" & " with \"pkg\"" % name, false) if name.toUpperAscii() in reservedNames: raiseNewValidationError( "\"$1\" is an invalid package name: reserved name" % name, false) proc validateVersion*(ver: string) = for c in ver: if c notin ({'.'} + Digits): raiseNewValidationError( "Version may only consist of numbers and the '.' character " & "but found '" & c & "'.", false) proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) = ## This ensures that a package's source code does not leak into ## another package's namespace. ## https://github.com/nim-lang/nimble/issues/144 let realDir = pkgInfo.getRealDir() normalizedBinNames = pkgInfo.bin.map( (x) => x.changeFileExt("").toLowerAscii() ) correctDir = if pkgInfo.name.toLowerAscii() in normalizedBinNames: pkgInfo.name & "pkg" else: pkgInfo.name proc onFile(path: string) = # Remove the root to leave only the package subdirectories. # ~/package-0.1/package/utils.nim -> package/utils.nim. var trailPath = changeRoot(realDir, "", path) if trailPath.startsWith(DirSep): trailPath = trailPath[1 .. ^1] let (dir, file, ext) = trailPath.splitFile # We're only interested in nim files, because only they can pollute our # namespace. if ext != (ExtSep & "nim"): return if dir.len == 0: if file != pkgInfo.name: # A source file was found in the top level of srcDir that doesn't share # a name with the package. let msg = ("Package '$1' has an incorrect structure. " & "The top level of the package source directory " & "should contain at most one module, " & "named '$2', but a file named '$3' was found. This " & "will be an error in the future.") % [pkgInfo.name, pkgInfo.name & ext, file & ext] hint = ("If this is the primary source file in the package, " & "rename it to '$1'. If it's a source file required by " & "the main module, or if it is one of several " & "modules exposed by '$4', then move it into a '$2' subdirectory. " & "If it's a test file or otherwise not required " & "to build the the package '$1', prevent its installation " & "by adding `skipFiles = @[\"$3\"]` to the .nimble file. See " & "https://github.com/nim-lang/nimble#libraries for more info.") % [pkgInfo.name & ext, correctDir & DirSep, file & ext, pkgInfo.name] raiseNewValidationError(msg, true, hint, true) else: assert(not pkgInfo.isMinimal) # On Windows `pkgInfo.bin` has a .exe extension, so we need to normalize. if not (dir.startsWith(correctDir & DirSep) or dir == correctDir): let msg = ("Package '$2' has an incorrect structure. " & "It should contain a single directory hierarchy " & "for source files, named '$3', but file '$1' " & "is in a directory named '$4' instead. " & "This will be an error in the future.") % [file & ext, pkgInfo.name, correctDir, dir] hint = ("If '$1' contains source files for building '$2', rename it " & "to '$3'. Otherwise, prevent its installation " & "by adding `skipDirs = @[\"$1\"]` to the .nimble file.") % [dir, pkgInfo.name, correctDir] raiseNewValidationError(msg, true, hint, true) iterInstallFiles(realDir, pkgInfo, options, onFile) proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) = let path = pkgInfo.myPath if pkgInfo.name == "": raiseNewValidationError("Incorrect .nimble file: " & path & " does not contain a name field.", false) if pkgInfo.name.normalize != path.splitFile.name.normalize: raiseNewValidationError( "The .nimble file name must match name specified inside " & path, true) if pkgInfo.version == "": raiseNewValidationError("Incorrect .nimble file: " & path & " does not contain a version field.", false) if not pkgInfo.isMinimal: if pkgInfo.author == "": raiseNewValidationError("Incorrect .nimble file: " & path & " does not contain an author field.", false) if pkgInfo.description == "": raiseNewValidationError("Incorrect .nimble file: " & path & " does not contain a description field.", false) if pkgInfo.license == "": raiseNewValidationError("Incorrect .nimble file: " & path & " does not contain a license field.", false) if pkgInfo.backend notin ["c", "cc", "objc", "cpp", "js"]: raiseNewValidationError("'" & pkgInfo.backend & "' is an invalid backend.", false) validatePackageStructure(pkginfo, options) proc nimScriptHint*(pkgInfo: PackageInfo) = if not pkgInfo.isNimScript: display("Warning:", "The .nimble file for this project could make use of " & "additional features, if converted into the new NimScript format." & "\nFor more details see:" & "https://github.com/nim-lang/nimble#creating-packages", Warning, HighPriority) proc multiSplit(s: string): seq[string] = ## Returns ``s`` split by newline and comma characters. ## ## Before returning, all individual entries are stripped of whitespace and ## also empty entries are purged from the list. If after all the cleanups are ## done no entries are found in the list, the proc returns a sequence with ## the original string as the only entry. result = split(s, {char(0x0A), char(0x0D), ','}) apply(result, proc(x: var string) = x = x.strip()) for i in countdown(result.len()-1, 0): if len(result[i]) < 1: result.del(i) # Huh, nothing to return? Return given input. if len(result) < 1: return @[s] proc readPackageInfoFromNimble(path: string; result: var PackageInfo) = var fs = newFileStream(path, fmRead) if fs != nil: var p: CfgParser open(p, fs, path) defer: close(p) var currentSection = "" while true: var ev = next(p) case ev.kind of cfgEof: break of cfgSectionStart: currentSection = ev.section of cfgKeyValuePair: case currentSection.normalize of "package": case ev.key.normalize of "name": result.name = ev.value of "version": result.version = ev.value of "author": result.author = ev.value of "description": result.description = ev.value of "license": result.license = ev.value of "srcdir": result.srcDir = ev.value of "bindir": result.binDir = ev.value of "skipdirs": result.skipDirs.add(ev.value.multiSplit) of "skipfiles": result.skipFiles.add(ev.value.multiSplit) of "skipext": result.skipExt.add(ev.value.multiSplit) of "installdirs": result.installDirs.add(ev.value.multiSplit) of "installfiles": result.installFiles.add(ev.value.multiSplit) of "installext": result.installExt.add(ev.value.multiSplit) of "bin": for i in ev.value.multiSplit: result.bin.add(i.addFileExt(ExeExt)) of "backend": result.backend = ev.value.toLowerAscii() case result.backend.normalize of "javascript": result.backend = "js" else: discard of "beforehooks": for i in ev.value.multiSplit: result.preHooks.incl(i.normalize) of "afterhooks": for i in ev.value.multiSplit: result.postHooks.incl(i.normalize) else: raise newException(NimbleError, "Invalid field: " & ev.key) of "deps", "dependencies": case ev.key.normalize of "requires": for v in ev.value.multiSplit: result.requires.add(parseRequires(v.strip)) else: raise newException(NimbleError, "Invalid field: " & ev.key) else: raise newException(NimbleError, "Invalid section: " & currentSection) of cfgOption: raise newException(NimbleError, "Invalid package info, should not contain --" & ev.value) of cfgError: raise newException(NimbleError, "Error parsing .nimble file: " & ev.msg) else: raise newException(ValueError, "Cannot open package info: " & path) proc readPackageInfoFromNims(scriptName: string, options: Options, result: var PackageInfo) = let iniFile = getIniFile(scriptName, options) if iniFile.fileExists(): readPackageInfoFromNimble(iniFile, result) proc inferInstallRules(pkgInfo: var PackageInfo, options: Options) = # Binary packages shouldn't install .nim files by default. # (As long as the package info doesn't explicitly specify what should be # installed.) let installInstructions = pkgInfo.installDirs.len + pkgInfo.installExt.len + pkgInfo.installFiles.len if installInstructions == 0 and pkgInfo.bin.len > 0: pkgInfo.skipExt.add("nim") # When a package doesn't specify a `srcDir` it's fair to assume that # the .nim files are in the root of the package. So we can explicitly select # them and prevent the installation of anything else. The user can always # override this with `installFiles`. if pkgInfo.srcDir == "": if dirExists(pkgInfo.getRealDir() / pkgInfo.name): pkgInfo.installDirs.add(pkgInfo.name) if fileExists(pkgInfo.getRealDir() / pkgInfo.name.addFileExt("nim")): pkgInfo.installFiles.add(pkgInfo.name.addFileExt("nim")) proc readPackageInfo(nf: NimbleFile, options: Options, onlyMinimalInfo=false): PackageInfo = ## Reads package info from the specified Nimble file. ## ## Attempts to read it using the "old" Nimble ini format first, if that ## fails attempts to evaluate it as a nimscript file. ## ## If both fail then returns an error. ## ## When ``onlyMinimalInfo`` is true, only the `name` and `version` fields are ## populated. The ``isNimScript`` field can also be relied on. ## ## This version uses a cache stored in ``options``, so calling it multiple ## times on the same ``nf`` shouldn't require re-evaluation of the Nimble ## file. assert fileExists(nf) # Check the cache. if options.pkgInfoCache.hasKey(nf): return options.pkgInfoCache[nf] result = initPackageInfo(nf) let minimalInfo = getNameVersion(nf) validatePackageName(nf.splitFile.name) var success = false var iniError: ref NimbleError # Attempt ini-format first. try: readPackageInfoFromNimble(nf, result) success = true result.isNimScript = false except NimbleError: iniError = (ref NimbleError)(getCurrentException()) if not success: if onlyMinimalInfo: result.name = minimalInfo.name result.version = minimalInfo.version result.isNimScript = true result.isMinimal = true # It's possible this proc will receive a .nimble-link file eventually, # I added this assert to hopefully make this error clear for everyone. let msg = "No version detected. Received nimble-link?" assert result.version.len > 0, msg else: try: readPackageInfoFromNims(nf, options, result) result.isNimScript = true except NimbleError as exc: if exc.hint.len > 0: raise let msg = "Could not read package info file in " & nf & ";\n" & " Reading as ini file failed with: \n" & " " & iniError.msg & ".\n" & " Evaluating as NimScript file failed with: \n" & " " & exc.msg & "." raise newException(NimbleError, msg) # By default specialVersion is the same as version. result.specialVersion = result.version # Only attempt to read a special version if `nf` is inside the $nimbleDir. if nf.startsWith(options.getNimbleDir()): # The package directory name may include a "special" version # (example #head). If so, it is given higher priority and therefore # overwrites the .nimble file's version. let version = parseVersionRange(minimalInfo.version) if version.kind == verSpecial: result.specialVersion = minimalInfo.version # Apply rules to infer which files should/shouldn't be installed. See #469. inferInstallRules(result, options) if not result.isMinimal: options.pkgInfoCache[nf] = result # Validate the rest of the package info last. if not options.disableValidation: validateVersion(result.version) validatePackageInfo(result, options) proc validate*(file: NimbleFile, options: Options, error: var ValidationError, pkgInfo: var PackageInfo): bool = try: pkgInfo = readPackageInfo(file, options) except ValidationError as exc: error = exc[] return false return true proc getPkgInfoFromFile*(file: NimbleFile, options: Options): PackageInfo = ## Reads the specified .nimble file and returns its data as a PackageInfo ## object. Any validation errors are handled and displayed as warnings. try: result = readPackageInfo(file, options) except ValidationError: let exc = (ref ValidationError)(getCurrentException()) if exc.warnAll: display("Warning:", exc.msg, Warning, HighPriority) display("Hint:", exc.hint, Warning, HighPriority) else: raise proc getPkgInfo*(dir: string, options: Options): PackageInfo = ## Find the .nimble file in ``dir`` and parses it, returning a PackageInfo. let nimbleFile = findNimbleFile(dir, true) return getPkgInfoFromFile(nimbleFile, options) proc getInstalledPkgs*(libsDir: string, options: Options): seq[tuple[pkginfo: PackageInfo, meta: MetaData]] = ## Gets a list of installed packages. ## ## ``libsDir`` is in most cases: ~/.nimble/pkgs/ const readErrorMsg = "Installed package '$1@$2' is outdated or corrupt." validationErrorMsg = readErrorMsg & "\nPackage did not pass validation: $3" hintMsg = "The corrupted package will need to be removed manually. To fix" & " this error message, remove $1." proc createErrorMsg(tmplt, path, msg: string): string = let (name, version) = getNameVersion(path) return tmplt % [name, version, msg] display("Loading", "list of installed packages", priority = MediumPriority) result = @[] for kind, path in walkDir(libsDir): if kind == pcDir: let nimbleFile = findNimbleFile(path, false) if nimbleFile != "": let meta = readMetaData(path) var pkg: PackageInfo try: pkg = readPackageInfo(nimbleFile, options, onlyMinimalInfo=false) except ValidationError: let exc = (ref ValidationError)(getCurrentException()) exc.msg = createErrorMsg(validationErrorMsg, path, exc.msg) exc.hint = hintMsg % path if exc.warnInstalled or exc.warnAll: display("Warning:", exc.msg, Warning, HighPriority) # Don't show hints here because they are only useful for package # owners. else: raise exc except: let tmplt = readErrorMsg & "\nMore info: $3" let msg = createErrorMsg(tmplt, path, getCurrentException().msg) var exc = newException(NimbleError, msg) exc.hint = hintMsg % path raise exc pkg.isInstalled = true pkg.isLinked = cmpPaths(nimbleFile.splitFile().dir, path) != 0 result.add((pkg, meta)) proc isNimScript*(nf: string, options: Options): bool = result = readPackageInfo(nf, options).isNimScript proc toFullInfo*(pkg: PackageInfo, options: Options): PackageInfo = if pkg.isMinimal: result = getPkgInfoFromFile(pkg.mypath, options) result.isInstalled = pkg.isInstalled result.isLinked = pkg.isLinked else: return pkg when isMainModule: validatePackageName("foo_bar") validatePackageName("f_oo_b_a_r") try: validatePackageName("foo__bar") assert false except NimbleError: assert true echo("Everything passed!")