Add sync2 mode, some random fixes

csproj.go → dirtyParsing.go

+// This file contains code to parse csproj, and packages.config.
+// It's dirty and hard to read.
 package main
 
 import (
@@ -6,14 +8,16 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
-	"regexp"
 	"strings"
 )
 
 type dependencyItem struct {
 	pkgName, envName, targetNetVer string
 }
+type dependencyItemStripped struct {
+	// This struct is intended to be passed to `nuget-download-package`, which accepts both pkgVer and targetNetVer.
+	pkgName, pkgVerOrNetVer string
+}
 
 // This function reads a parsed XML root node, and guess the target dotnet version of this CSPROJ.
 // This function is only for internal usage.
@@ -108,23 +112,6 @@ func ParseDependencies(csprojPath string) (dependencies []dependencyItem, err er
 	return
 }
 
-// List all `.csproj` file in directory, recursively.
-func ScanCsprojInDirectory(dirPath string) (csprojPathes []string, err error) {
-	libRegEx, err := regexp.Compile("^.+\\.(csproj)$")
-	if err != nil {
-		return
-	}
-
-	err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
-		if err == nil && libRegEx.MatchString(info.Name()) {
-			csprojPathes = append(csprojPathes, path)
-		}
-		return nil
-	})
-
-	return
-}
-
 // This function panic on error, because it's not recoverable.
 // Returns key-value pairs for environment variables to set.
 func GenerateCorextEnvvar(localRepoPath, buildOutputPath string, deps []dependencyItem) map[string]string {
@@ -159,3 +146,38 @@ func GenerateCorextEnvvar(localRepoPath, buildOutputPath string, deps []dependen
 
 	return result
 }
+
+// Parse packages.config for pkgName:pkgVer pair. The returned dependencyItem contains pkgVer instead of netVer.
+func ParsePackagesConfig(packagesConfigPath string) (dependencies []dependencyItemStripped, err error) {
+	f, err := os.Open(packagesConfigPath)
+	if err != nil {
+		return
+	}
+	parsed, err := xmlquery.Parse(f)
+	if err != nil {
+		return
+	}
+
+	packages := xmlquery.Find(parsed, "//packages/package")
+	for _, pkgEntry := range packages {
+		pkgName, pkgVer := "", ""
+		for _, attr := range pkgEntry.Attr {
+			switch attr.Name.Local {
+			case "id":
+				pkgName = attr.Value
+			case "version":
+				pkgVer = attr.Value
+			}
+		}
+
+		if pkgName == "" || pkgVer == "" {
+			log.Println("Error: bad packages.config entry (unable to locate `id` or `version`): " + pkgEntry.OutputXML(true))
+			continue
+		}
+
+		dependencies = append(dependencies, dependencyItemStripped{pkgName, pkgVer})
+	}
+	return
+}
+
+

main.go

@@ -10,13 +10,18 @@ import (
 // Some options here. Would be improved in beta release.
 const DEDUCT_PKGNAME_FROM_VARNAME = true
 const USE_PROJECT_NETVER_INSTEAD_OF_HINTPATH_NETVER = false
-const OPENXT_VERSION = "1.3-2"
+const OPENXT_VERSION = "1.4.1-1"
 
 func print_help_and_exit() {
 	println("Usage: openxt <subcommand> [options...]")
-	println("Subcommand := sync / env")
+	println("Subcommand := sync / sync2 / env")
 	println("For 'sync', you must set --local-repo-dir, --project-dir")
+	println("For 'sync2', you must set --local-repo-dir, --project-dir")
 	println("For 'env', you must set --local-repo-dir, --project-dir, --bin-dir")
+	println("")
+    println("'sync' would guess package version from your csproj, while 'sync2' would only use package version from packages.config. ")
+	println("It doesn't mean that sync2 is better than sync. Both works well in well-written projects. ")
+	println("Guideline: If upstream nuget packager sucks, use 'sync2'. If csproj coder sucks, use 'sync'. ")
 	flag.PrintDefaults()
 	os.Exit(1)
 }
@@ -39,18 +44,22 @@ func main() {
 
 	assertStringNotEmpty(*projectDir, "`--project-dir` must be set")
 	assertStringNotEmpty(*localRepoDir, "`--local-repo-dir` must be set")
-	deps := ScanForDependencies(*projectDir)
 
 	switch os.Args[0] {
 	case "sync":
+		deps := ScanForDependencies(*projectDir)
 		SyncPackages(*nugetConfigPath, *localRepoDir, deps)
+	case "sync2":
+		deps2 := ScanForDependencies2(*projectDir)
+		SyncPackages2(*nugetConfigPath, *localRepoDir, deps2)
 	case "env":
 		assertStringNotEmpty(*binDir, "`--bin-dir` must be set")
 
+		deps := ScanForDependencies(*projectDir)
 		envvar := GenerateCorextEnvvar(*localRepoDir, *binDir, deps)
 		// log.Println() and println() writes to stderr, and fmt.Println() writes to stdout.
 		fmt.Println(SerializeEnvvar(*shell, false, envvar))
 	default:
-		panic("Expected subcommand `sync` or `env`. ")
+		panic("Expected subcommand `sync`, `sync2` or `env`. ")
 	}
 }

packageManager.go

@@ -13,10 +13,11 @@ import (
 
 // Scan a directory recursively, and parse `**/*.csproj`, to list all referenced nuget packages. (in CoreXT reference format)
 func ScanForDependencies(scanPath string) (allDeps []dependencyItem) {
-	// Scan for dependencies
-	files, err := ScanCsprojInDirectory(scanPath)
+	// Firstly: Find all .csproj files, recursively.
+	files, err := FindInDirectory(scanPath, "^.+\\.(csproj)$")
 	logErrorIfAny(err, "ScanCsprojInDirectory")
 
+	// Secondly: parse them for deps.
 	for _, file := range files {
 		dependencies, err := ParseDependencies(file)
 		logErrorIfAny(err, "ParseDependencies " + file)
@@ -24,41 +25,71 @@ func ScanForDependencies(scanPath string) (allDeps []dependencyItem) {
 	}
 	return
 }
+// Scan a directory recursively, and parse `**/packages.config`, to list all referenced nuget packages. (in nuget standard format)
+func ScanForDependencies2(scanPath string) (allDeps []dependencyItemStripped) {
+	// Firstly, scan for all packages.config, recursively.
+	files, err := FindInDirectory(scanPath, "^packages.config$")
+	logErrorIfAny(err, "Scan packages.config in directory")
+
+	// Secondly, parse them for deps.
+	for _, file := range files {
+		log.Printf("Downloading packages from %v", file)
+		deps, err := ParsePackagesConfig(file)
+		logErrorIfAny(err, "Parse packages config")
+
+		allDeps = append(allDeps, deps...)
+	}
+	return
+}
+
 
 // Download `allDeps` to localRepoPath, with nuget.config from nugetConfigPath.
 // However, nugetConfigPath is not supported now. We use default nuget.config now. (Usually in ~/.nuget/)
+// Update: nugetConfigPath is support after migrated to nuget-download-package.
 func SyncPackages(nugetConfigPath, localRepoPath string, allDeps []dependencyItem) {
-	// Prepare nuget local repo
-	depsToSync := depsDeduplicate(allDeps)
+	allDeps = depsDeduplicate(allDeps)
+
+	// Convert dependencyItem(for CoreXT) to dependencyItemStripped(for nuget-package-download)
+	depsStripped := []dependencyItemStripped{}
+	for _, dep := range allDeps {
+		realPkgName, pkgVerOrNetVer := extractPostfixPkgVerFromPkgName(dep.pkgName)
+		if pkgVerOrNetVer == "" {
+			// This is not a pkgname.pkgver format. Use targetNetVer to deduce.
+			pkgVerOrNetVer = dep.targetNetVer
+		}
+		depsStripped = append(depsStripped, dependencyItemStripped{realPkgName, pkgVerOrNetVer})
+	}
+
+	SyncPackages2(nugetConfigPath, localRepoPath, depsStripped)
+}
+
+// This function does the actual job, to download packages.
+func SyncPackages2(nugetConfigPath, localRepoPath string, allDeps []dependencyItemStripped) {
+	allDeps = depsDeduplicate2(allDeps)
 
 	var wg sync.WaitGroup
 	concurrencyLimitChan := make(chan int, 64)
-	for index_, dep_ := range depsToSync {
+	for index_, dep_ := range allDeps {
 		wg.Add(1)
 		concurrencyLimitChan <- 1 // will block if there's already 64 goroutine running
-		go func(index int, dep dependencyItem) {
+		go func(index int, dep dependencyItemStripped) {
 			defer func() {<-concurrencyLimitChan ; wg.Done()}()
-			log.Printf("[%v/%v] Begin downloading: %v:%v as var %v", index, len(depsToSync), dep.pkgName, dep.targetNetVer, dep.envName)
-
-			realPkgName, realPkgVer := extractPostfixPkgVerFromPkgName(dep.pkgName)
-			if realPkgVer == "" {
-				// This is not a pkgname.pkgver format. Use targetNetVer to deduce.
-				realPkgVer = dep.targetNetVer
-			}
-
-			log.Println("EXEC: nuget-download-package", realPkgName, realPkgVer, localRepoPath, nugetConfigPath)
-			cmd := exec.Command("nuget-download-package", realPkgName, realPkgVer, localRepoPath, nugetConfigPath)
+			log.Printf("[%v/%v] Begin downloading: %v:%v", index, len(allDeps), dep.pkgName, dep.pkgVerOrNetVer)
+			log.Println("EXEC: nuget-download-package", dep.pkgName, dep.pkgVerOrNetVer, localRepoPath, nugetConfigPath)
+			cmd := exec.Command("nuget-download-package", dep.pkgName, dep.pkgVerOrNetVer, localRepoPath, nugetConfigPath)
 			stdout, err := cmd.CombinedOutput()
 			log.Println(string(stdout))
 			// We can do nothing on failure. usually, if envName is NULL, we need not install it at all.
 			logErrorIfAny(err, "EXEC command")
 
-			log.Printf("[%v/%v] End downloading: %v:%v as var %v", index, len(depsToSync), dep.pkgName, dep.targetNetVer, dep.envName)
+			log.Printf("[%v/%v] End downloading: %v:%v", index, len(allDeps), dep.pkgName, dep.pkgVerOrNetVer)
 		}(index_, dep_)
 	}
 	wg.Wait()
 }
 
+
+
 // Given package name, this function lookup the localRepoPath, and returns path to a good package version.
 // The returned pkgPath is used to populate environment variables.
 // |

tools/nuget-download-package/main.go

+// This file contains very dirty code to parse the noob nuget response.
+// Be careful before touching it!
 package main
 
 import (
@@ -308,27 +310,39 @@ func main() {
 		nugetConfigPath = os.Args[4]
 	}
 
+	if len(pkgVerOrNetVer) > 3 && pkgVerOrNetVer[:3] != "net" {
+		// If requested version already exists, just skip and exit.
+		pkgBaseDir := filepath.Join(localRepoDir, pkgName, pkgVerOrNetVer)
+		if stat, err := os.Stat(filepath.Join(pkgBaseDir, "lib")); err == nil && stat.IsDir() {
+			log.Println(pkgBaseDir + "/lib already exists. This version has already been installed. Skipping the installation. ")
+			return
+		}
+	}
+
+	// Download the index. TODO: cache this index across multiple runs.
     indexJsons, indexJsons_Login := initNugetIndexJsonCache(nugetConfigPath, pkgName)
 
+    // Decide a good package version to download.
 	pkgVer, targetUrl, targetUrlLogin := "", "", ""
     if strings.HasPrefix(pkgVerOrNetVer, "net") {
     	pkgVer, targetUrl, targetUrlLogin = decidePackageVersion(pkgVerOrNetVer, "", indexJsons, indexJsons_Login, os.Getenv("OPENXT_DEBUG") == "1")
     } else {
 		pkgVer, targetUrl, targetUrlLogin = decidePackageVersion("", pkgVerOrNetVer, indexJsons, indexJsons_Login, os.Getenv("OPENXT_DEBUG") == "1")
 	}
-
 	if pkgVer == "" {
 		log.Println("Error: can not find a valid pkgVer for " + pkgName + ":" + pkgVerOrNetVer)
 		os.Exit(2)
 	}
 	log.Println("Using pkgVer " + pkgVer + " from " + targetUrl)
 
+	// If requested version already exists, just skip and exit.
 	pkgBaseDir := filepath.Join(localRepoDir, pkgName, pkgVer)
 	if stat, err := os.Stat(filepath.Join(pkgBaseDir, "lib")); err == nil && stat.IsDir() {
 		log.Println(pkgBaseDir + "/lib already exists. This version has already been installed. Skipping the installation. ")
 		return
 	}
 
+	// Download the package and extract it.
 	err := os.RemoveAll(pkgBaseDir)
 	logErrorIfAny(err, "rm -rf " + pkgBaseDir)
 	err = os.MkdirAll(pkgBaseDir, 0777)

util.go

@@ -4,7 +4,9 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"path/filepath"
 	"reflect"
+	"regexp"
 	"strings"
 )
 
@@ -92,6 +94,17 @@ func depsDeduplicate(deps []dependencyItem) (resultDeps []dependencyItem) {
 	}
 	return
 }
+func depsDeduplicate2(deps []dependencyItemStripped) (resultDeps []dependencyItemStripped) {
+	hashset := make(map[string]bool)
+	for _, dep := range deps {
+		niddle := dep.pkgName + dep.pkgVerOrNetVer
+		if _, ok := hashset[niddle]; !ok {
+			hashset[niddle] = true
+			resultDeps = append(resultDeps, dep)
+		}
+	}
+	return
+}
 
 // Sometimes, they use `PkgMicrosoft_VisualStudio_Services_Release_Client_15_131_1` as var name,
 // which would yeild Microsoft.VisualStudio.Services.Release.Client.15.131.1 as package name.
@@ -128,3 +141,25 @@ func extractPostfixPkgVerFromPkgName(badPkgName string) (realPkgName, pkgVer str
 	}
 	return
 }
+
+// List file in directory, recursively.
+// `find $dirPath -name "$regexToFind"`
+// Example: find all .csproj: "^.+\\.(csproj)$"
+func FindInDirectory(dirPath, regexToFind string) (resultFilePathes []string, err error) {
+	libRegEx, err := regexp.Compile(regexToFind)
+	if err != nil {
+		return
+	}
+
+	err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
+		if err == nil && libRegEx.MatchString(info.Name()) {
+			resultFilePathes = append(resultFilePathes, path)
+		}
+		return nil
+	})
+
+	return
+}
+
+
+