From 06a94ba28f1e021937b94b5f1995ba5ad98d6122 Mon Sep 17 00:00:00 2001 From: Yuriy Glukhov Date: Mon, 7 Aug 2017 17:22:47 +0300 Subject: [PATCH] Lower memory consumption --- src/nimble.nim | 2 +- src/nimblepkg/options.nim | 5 ++ src/nimblepkg/packageinfo.nim | 40 +++++++-------- src/nimblepkg/packageparser.nim | 86 ++++++++++++++++----------------- 4 files changed, 69 insertions(+), 64 deletions(-) diff --git a/src/nimble.nim b/src/nimble.nim index 6770694..7321a96 100644 --- a/src/nimble.nim +++ b/src/nimble.nim @@ -445,7 +445,7 @@ proc installFromDir(dir: string, requestedVer: VersionRange, options: Options, createDir(pkgDestDir) # Copy this package's files based on the preferences specified in PkgInfo. var filesInstalled = initSet[string]() - for file in getInstallFiles(realDir, pkgInfo, options): + discard forEachInstallFile(realDir, pkgInfo, options) do(file: string) -> bool: createDir(changeRoot(realDir, pkgDestDir, file.splitFile.dir)) let dest = changeRoot(realDir, pkgDestDir, file) filesInstalled.incl copyFileD(file, dest) diff --git a/src/nimblepkg/options.nim b/src/nimblepkg/options.nim index d68e524..eee40cd 100644 --- a/src/nimblepkg/options.nim +++ b/src/nimblepkg/options.nim @@ -21,6 +21,7 @@ type showHelp*: bool showVersion*: bool noColor*: bool + disableValidation*: bool ActionType* = enum actionNil, actionRefresh, actionInit, actionDump, actionPublish, @@ -107,6 +108,9 @@ proc writeHelp*(quit=true) = proc writeVersion*() = echo("nimble v$# compiled at $# $#" % [nimbleVersion, CompileDate, CompileTime]) + const gitVersion = staticExec("git rev-parse HEAD") + when gitVersion.len > 0: + echo "git hash: ", gitVersion raise NimbleQuit(msg: "") proc parseActionType*(action: string): ActionType = @@ -252,6 +256,7 @@ proc parseFlag*(flag, val: string, result: var Options, kind = cmdLongOption) = of "verbose": result.verbosity = LowPriority of "debug": result.verbosity = DebugPriority of "nocolor": result.noColor = true + of "disablevalidation": result.disableValidation = true # Action-specific flags. else: case result.action.typ diff --git a/src/nimblepkg/packageinfo.nim b/src/nimblepkg/packageinfo.nim index e85fcd1..fcfead6 100644 --- a/src/nimblepkg/packageinfo.nim +++ b/src/nimblepkg/packageinfo.nim @@ -430,30 +430,30 @@ proc checkInstallDir(pkgInfo: PackageInfo, if thisDir[0] == '.': result = true if thisDir == "nimcache": result = true -proc findWithExt(dir: string, pkgInfo: PackageInfo): seq[string] = - ## Returns the filenames of the files that should be copied. - result = @[] +proc forEachFileWithExt(dir: string, pkgInfo: PackageInfo, + action: proc(f: string): bool): bool = + ## Runs `action` for each filename of the files that should be copied. + ## Stops if `action` returns `true`. for kind, path in walkDir(dir): if kind == pcDir: - result.add findWithExt(path, pkgInfo) + if forEachFileWithExt(path, pkgInfo, action): return true else: if path.splitFile.ext[1 .. ^1] in pkgInfo.installExt: - result.add path + if action(path): return true -proc getFilesInDir(dir: string): seq[string] = - ## Returns a list of paths to files inside the specified directory and any - ## subdirectories that are in it. - result = @[] +proc forEachFileInDir(dir: string, action: proc(f: string): bool): bool = + ## Runs `action` for each file in ``dir`` and any + ## subdirectories that are in it. Stops if `action` returns `true`. for kind, path in walkDir(dir): if kind == pcDir: - result.add getFilesInDir(path) + if forEachFileInDir(path, action): return true else: - result.add path + if action(path): return true -proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo, - options: Options): seq[string] = - ## Returns a list of files within the ``realDir`` that should be installed. - result = @[] +proc forEachInstallFile*(realDir: string, pkgInfo: PackageInfo, + options: Options, action: proc(f: string): bool): bool = + ## Runs `action` for each file within the ``realDir`` that should be installed. + ## Stops if `action` returns `true`. let whitelistMode = pkgInfo.installDirs.len != 0 or pkgInfo.installFiles.len != 0 or @@ -466,7 +466,7 @@ proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo, continue else: raise NimbleQuit(msg: "") - result.add src + if action(src): return true for dir in pkgInfo.installDirs: # TODO: Allow skipping files inside dirs? @@ -477,9 +477,9 @@ proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo, else: raise NimbleQuit(msg: "") - result.add getFilesInDir(src) + if forEachFileInDir(src, action): return true - result.add findWithExt(realDir, pkgInfo) + if forEachFileWithExt(realDir, pkgInfo, action): return true else: for kind, file in walkDir(realDir): if kind == pcDir: @@ -487,13 +487,13 @@ proc getInstallFiles*(realDir: string, pkgInfo: PackageInfo, if skip: continue - result.add getInstallFiles(file, pkgInfo, options) + if forEachInstallFile(file, pkgInfo, options, action): return true else: let skip = pkgInfo.checkInstallFile(realDir, file) if skip: continue - result.add file + if action(file): return true when isMainModule: doAssert getNameVersion("/home/user/.nimble/libs/packagea-0.1") == diff --git a/src/nimblepkg/packageparser.nim b/src/nimblepkg/packageparser.nim index 37bb679..2fddfa9 100644 --- a/src/nimblepkg/packageparser.nim +++ b/src/nimblepkg/packageparser.nim @@ -79,7 +79,8 @@ proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) = pkgInfo.name & "pkg" else: pkgInfo.name - for path in getInstallFiles(realDir, pkgInfo, options): + + discard forEachInstallFile(realDir, pkgInfo, options) do(path: string) -> bool: # Remove the root to leave only the package subdirectories. # ~/package-0.1/package/utils.nim -> package/utils.nim. var trailPath = changeRoot(realDir, "", path) @@ -87,46 +88,44 @@ proc validatePackageStructure(pkgInfo: PackageInfo, options: Options) = let (dir, file, ext) = trailPath.splitFile # We're only interested in nim files, because only they can pollute our # namespace. - if ext != (ExtSep & "nim"): - continue - - if dir.len == 0: - if file != pkgInfo.name: - # A source file was found in the top level of srcDir that doesn't share - # a name with the package. - let - msg = ("Package '$1' has an incorrect structure. " & - "The top level of the package source directory " & - "should contain at most one module, " & - "named '$2', but a file named '$3' was found. This " & - "will be an error in the future.") % - [pkgInfo.name, pkgInfo.name & ext, file & ext] - hint = ("If this is the primary source file in the package, " & - "rename it to '$1'. If it's a source file required by " & - "the main module, or if it is one of several " & - "modules exposed by '$4', then move it into a '$2' subdirectory. " & - "If it's a test file or otherwise not required " & - "to build the the package '$1', prevent its installation " & - "by adding `skipFiles = @[\"$3\"]` to the .nimble file. See " & - "https://github.com/nim-lang/nimble#libraries for more info.") % - [pkgInfo.name & ext, correctDir & DirSep, file & ext, pkgInfo.name] - raiseNewValidationError(msg, true, hint, true) - else: - assert(not pkgInfo.isMinimal) - # On Windows `pkgInfo.bin` has a .exe extension, so we need to normalize. - if not (dir.startsWith(correctDir & DirSep) or dir == correctDir): - let - msg = ("Package '$2' has an incorrect structure. " & - "It should contain a single directory hierarchy " & - "for source files, named '$3', but file '$1' " & - "is in a directory named '$4' instead. " & - "This will be an error in the future.") % - [file & ext, pkgInfo.name, correctDir, dir] - hint = ("If '$1' contains source files for building '$2', rename it " & - "to '$3'. Otherwise, prevent its installation " & - "by adding `skipDirs = @[\"$1\"]` to the .nimble file.") % - [dir, pkgInfo.name, correctDir] - raiseNewValidationError(msg, true, hint, true) + if ext == (ExtSep & "nim"): + if dir.len == 0: + if file != pkgInfo.name: + # A source file was found in the top level of srcDir that doesn't share + # a name with the package. + let + msg = ("Package '$1' has an incorrect structure. " & + "The top level of the package source directory " & + "should contain at most one module, " & + "named '$2', but a file named '$3' was found. This " & + "will be an error in the future.") % + [pkgInfo.name, pkgInfo.name & ext, file & ext] + hint = ("If this is the primary source file in the package, " & + "rename it to '$1'. If it's a source file required by " & + "the main module, or if it is one of several " & + "modules exposed by '$4', then move it into a '$2' subdirectory. " & + "If it's a test file or otherwise not required " & + "to build the the package '$1', prevent its installation " & + "by adding `skipFiles = @[\"$3\"]` to the .nimble file. See " & + "https://github.com/nim-lang/nimble#libraries for more info.") % + [pkgInfo.name & ext, correctDir & DirSep, file & ext, pkgInfo.name] + raiseNewValidationError(msg, true, hint, true) + else: + assert(not pkgInfo.isMinimal) + # On Windows `pkgInfo.bin` has a .exe extension, so we need to normalize. + if not (dir.startsWith(correctDir & DirSep) or dir == correctDir): + let + msg = ("Package '$2' has an incorrect structure. " & + "It should contain a single directory hierarchy " & + "for source files, named '$3', but file '$1' " & + "is in a directory named '$4' instead. " & + "This will be an error in the future.") % + [file & ext, pkgInfo.name, correctDir, dir] + hint = ("If '$1' contains source files for building '$2', rename it " & + "to '$3'. Otherwise, prevent its installation " & + "by adding `skipDirs = @[\"$1\"]` to the .nimble file.") % + [dir, pkgInfo.name, correctDir] + raiseNewValidationError(msg, true, hint, true) proc validatePackageInfo(pkgInfo: PackageInfo, options: Options) = let path = pkgInfo.myPath @@ -317,8 +316,9 @@ proc readPackageInfo(nf: NimbleFile, options: Options, options.pkgInfoCache[nf] = result # Validate the rest of the package info last. - validateVersion(result.version) - validatePackageInfo(result, options) + if not options.disableValidation: + validateVersion(result.version) + validatePackageInfo(result, options) proc getPkgInfoFromFile*(file: NimbleFile, options: Options): PackageInfo = ## Reads the specified .nimble file and returns its data as a PackageInfo