From 78b69bbdaba682bd4ab0b6dc11f7fc4553e7d534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Sun, 7 Feb 2021 17:55:53 +0000 Subject: [PATCH] share a single temporary directory between all processes Each compile and link sub-process created its own temporary directory, to be cleaned up shortly after. Moreover, we also had the global gob-encoded temporary file. Instead, place all of those under a single, start-to-end temporary directory. This is cleaner for the end user, and easier to maintain for us. A big plus is that we can also get rid of the confusing deferred global, as it was mostly used to clean up these extra temp dirs. The only remaining use was post-compile code, which is now an explicit func returned by each "transform" func. While at it, clean up the math/rand seeding code a bit and add a debug log line, and stop shadowing a cmd string with a cmd *exec.Cmd. Fixes #147. --- import_obfuscation.go | 4 +- main.go | 131 ++++++++++++++++++++---------------------- shared.go | 18 +++--- 3 files changed, 73 insertions(+), 80 deletions(-) diff --git a/import_obfuscation.go b/import_obfuscation.go index 58a4ac1..bcf2477 100644 --- a/import_obfuscation.go +++ b/import_obfuscation.go @@ -153,7 +153,7 @@ func extractDebugObfSrc(pkgPath string, pkg *goobj2.Package) error { // It returns the path to the modified main object file, to be used for linking. // We also return a map of how the imports were garbled, as well as the private // name map recovered from the archive files, so that we can amend -X flags. -func obfuscateImports(objPath, tempDir string, importMap goobj2.ImportMap) (garbledObj string, garbledImports, privateNameMap map[string]string, _ error) { +func obfuscateImports(objPath string, importMap goobj2.ImportMap) (garbledObj string, garbledImports, privateNameMap map[string]string, _ error) { mainPkg, err := goobj2.Parse(objPath, "main", importMap) if err != nil { return "", nil, nil, fmt.Errorf("error parsing main objfile: %v", err) @@ -274,7 +274,7 @@ func obfuscateImports(objPath, tempDir string, importMap goobj2.ImportMap) (garb // An archive under the temporary file. Note that // ioutil.TempFile creates a file to ensure no collisions, so we // simply use its name after closing the file. - tempObjFile, err := ioutil.TempFile(tempDir, "pkg.*.a") + tempObjFile, err := ioutil.TempFile(sharedTempDir, "pkg.*.a") if err != nil { return "", nil, nil, fmt.Errorf("creating temp file: %v", err) } diff --git a/main.go b/main.go index 9aa2c75..33e5a36 100644 --- a/main.go +++ b/main.go @@ -84,8 +84,8 @@ For more information, see https://github.com/burrowers/garble. func main() { os.Exit(main1()) } var ( - deferred []func() error - fset = token.NewFileSet() + fset = token.NewFileSet() + sharedTempDir = os.Getenv("GARBLE_SHARED") nameCharset = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789_z" b64 = base64.NewEncoding(nameCharset) @@ -247,7 +247,7 @@ How to install Go: https://golang.org/doc/install func mainErr(args []string) error { // If we recognize an argument, we're not running within -toolexec. - switch cmd, args := args[0], args[1:]; cmd { + switch command, args := args[0], args[1:]; command { case "help": return flag.ErrHelp case "version": @@ -286,7 +286,7 @@ func mainErr(args []string) error { // Note that we also need to pass build flags to 'go list', such // as -tags. cache.BuildFlags = filterBuildFlags(flags) - if cmd == "test" { + if command == "test" { cache.BuildFlags = append(cache.BuildFlags, "-test") } @@ -302,18 +302,18 @@ func mainErr(args []string) error { return err } - sharedName, err := saveShared() - if err != nil { + if sharedTempDir, err = saveShared(); err != nil { return err } - defer os.Remove(sharedName) + os.Setenv("GARBLE_SHARED", sharedTempDir) + defer os.Remove(sharedTempDir) goArgs := []string{ - cmd, + command, "-trimpath", "-toolexec=" + cache.ExecPath, } - if cmd == "test" { + if command == "test" { // vet is generally not useful on garbled code; keep it // disabled by default. goArgs = append(goArgs, "-vet=off") @@ -326,12 +326,16 @@ func mainErr(args []string) error { cmd.Stderr = os.Stderr return cmd.Run() } + if !filepath.IsAbs(args[0]) { // -toolexec gives us an absolute path to the tool binary to // run, so this is most likely misuse of garble by a user. return fmt.Errorf("unknown command: %q", args[0]) } + // We're in a toolexec sub-process, not directly called by the user. + // Load the shared data and wrap the tool, like the compiler or linker. + if err := loadShared(); err != nil { return err } @@ -347,35 +351,34 @@ func mainErr(args []string) error { transform := transformFuncs[tool] transformed := args[1:] + var postFunc func() error // log.Println(tool, transformed) if transform != nil { var err error - if transformed, err = transform(transformed); err != nil { + if transformed, postFunc, err = transform(transformed); err != nil { return err } } - defer func() { - for _, fn := range deferred { - if err := fn(); err != nil { - fmt.Fprintln(os.Stderr, err) - } - } - }() cmd := exec.Command(args[0], transformed...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr if err := cmd.Run(); err != nil { return err } + if postFunc != nil { + if err := postFunc(); err != nil { + return err + } + } return nil } -var transformFuncs = map[string]func([]string) ([]string, error){ +var transformFuncs = map[string]func([]string) (args []string, post func() error, _ error){ "compile": transformCompile, "link": transformLink, } -func transformCompile(args []string) ([]string, error) { +func transformCompile(args []string) ([]string, func() error, error) { var err error flags, paths := splitFlagsFromFiles(args, ".go") @@ -391,7 +394,7 @@ func transformCompile(args []string) ([]string, error) { opts.GarbleLiterals = false opts.DebugDir = "" } else if !isPrivate(curPkgPath) { - return append(flags, paths...), nil + return append(flags, paths...), nil, nil } for i, path := range paths { if filepath.Base(path) == "_gomod_.go" { @@ -401,34 +404,35 @@ func transformCompile(args []string) ([]string, error) { } } if len(paths) == 1 && filepath.Base(paths[0]) == "_testmain.go" { - return append(flags, paths...), nil + return append(flags, paths...), nil, nil } // If the value of -trimpath doesn't contain the separator ';', the 'go // build' command is most likely not using '-trimpath'. trimpath := flagValue(flags, "-trimpath") if !strings.Contains(trimpath, ";") { - return nil, fmt.Errorf("-toolexec=garble should be used alongside -trimpath") + return nil, nil, fmt.Errorf("-toolexec=garble should be used alongside -trimpath") } if err := fillBuildInfo(flags); err != nil { - return nil, err + return nil, nil, err } var files []*ast.File for _, path := range paths { file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) if err != nil { - return nil, err + return nil, nil, err } files = append(files, file) } - if len(opts.Seed) > 0 { - mathrand.Seed(int64(binary.BigEndian.Uint64(opts.Seed))) - } else { - mathrand.Seed(int64(binary.BigEndian.Uint64([]byte(curActionID)))) + randSeed := opts.Seed + if len(randSeed) == 0 { + randSeed = curActionID } + // log.Printf("seeding math/rand with %x\n", randSeed) + mathrand.Seed(int64(binary.BigEndian.Uint64(randSeed))) tf := &transformer{ info: &types.Info{ @@ -441,7 +445,7 @@ func transformCompile(args []string) ([]string, error) { origTypesConfig := types.Config{Importer: origImporter} tf.pkg, err = origTypesConfig.Check(curPkgPath, fset, files, tf.info) if err != nil { - return nil, fmt.Errorf("typecheck error: %v", err) + return nil, nil, fmt.Errorf("typecheck error: %v", err) } tf.privateNameMap = make(map[string]string) @@ -453,18 +457,10 @@ func transformCompile(args []string) ([]string, error) { files = literals.Obfuscate(files, tf.info, fset, tf.ignoreObjects) } - tempDir, err := ioutil.TempDir("", "garble-build") - if err != nil { - return nil, err - } - deferred = append(deferred, func() error { - return os.RemoveAll(tempDir) - }) - // Add our temporary dir to the beginning of -trimpath, so that we don't // leak temporary dirs. Needs to be at the beginning, since there may be // shorter prefixes later in the list, such as $PWD if TMPDIR=$PWD/tmp. - flags = flagSetValue(flags, "-trimpath", tempDir+"=>;"+trimpath) + flags = flagSetValue(flags, "-trimpath", sharedTempDir+"=>;"+trimpath) // log.Println(flags) detachedComments := make([][]string, len(files)) @@ -520,12 +516,12 @@ func transformCompile(args []string) ([]string, error) { // Uncomment for some quick debugging. Do not delete. // fmt.Fprintf(os.Stderr, "\n-- %s/%s --\n", curPkgPath, origName) // if err := printConfig.Fprint(os.Stderr, fset, file); err != nil { - // return nil, err + // return nil, nil, err // } } - tempFile, err := os.Create(filepath.Join(tempDir, name)) + tempFile, err := ioutil.TempFile(sharedTempDir, name+".*.go") if err != nil { - return nil, err + return nil, nil, err } defer tempFile.Close() @@ -534,14 +530,14 @@ func transformCompile(args []string) ([]string, error) { for _, comment := range detachedComments[i] { if _, err := printWriter.Write([]byte(comment + "\n")); err != nil { - return nil, err + return nil, nil, err } } if err := printConfig.Fprint(printWriter, fset, file); err != nil { - return nil, err + return nil, nil, err } if err := tempFile.Close(); err != nil { - return nil, err + return nil, nil, err } if err := obfSrcTarWriter.WriteHeader(&tar.Header{ @@ -550,17 +546,18 @@ func transformCompile(args []string) ([]string, error) { ModTime: time.Now(), // Need for restoring obfuscation time Size: int64(obfSrc.Len()), }); err != nil { - return nil, err + return nil, nil, err } if _, err := obfSrcTarWriter.Write(obfSrc.Bytes()); err != nil { - return nil, err + return nil, nil, err } newPaths = append(newPaths, tempFile.Name()) } + // After the compilation succeeds, add our headers to the object file. objPath := flagValue(flags, "-o") - deferred = append(deferred, func() error { + postCompile := func() error { importMap := func(importPath string) (objectPath string) { return buildInfo.imports[importPath].packagefile } @@ -577,24 +574,23 @@ func transformCompile(args []string) ([]string, error) { // Adding an extra archive header is safe, // and shouldn't break other tools like the linker since our header name is unique - pkg.ArchiveMembers = append(pkg.ArchiveMembers, goobj2.ArchiveMember{ - ArchiveHeader: goobj2.ArchiveHeader{ + pkg.ArchiveMembers = append(pkg.ArchiveMembers, + goobj2.ArchiveMember{ArchiveHeader: goobj2.ArchiveHeader{ Name: garbleMapHeaderName, Size: int64(len(data)), Data: data, - }, - }, goobj2.ArchiveMember{ - ArchiveHeader: goobj2.ArchiveHeader{ + }}, + goobj2.ArchiveMember{ArchiveHeader: goobj2.ArchiveHeader{ Name: garbleSrcHeaderName, Size: int64(obfSrcArchive.Len()), Data: obfSrcArchive.Bytes(), - }, - }) + }}, + ) return pkg.Write(objPath) - }) + } - return append(flags, newPaths...), nil + return append(flags, newPaths...), postCompile, nil } // handleDirectives looks at all the comments in a file containing build @@ -1105,6 +1101,9 @@ func (tf *transformer) transformGo(file *ast.File) *ast.File { // "if" branch below. Plus, if this edge case triggers // multiple times in a single package compile, we can // call "go list" once and cache its result. + if pkg.ImportPath != path { + panic(fmt.Sprintf("unexpected path: %q vs %q", pkg.ImportPath, path)) + } buildInfo.imports[path] = importedPkg{ packagefile: pkg.Export, actionID: decodeHash(splitActionID(buildID)), @@ -1219,34 +1218,26 @@ func isTestSignature(sign *types.Signature) bool { return obj != nil && obj.Pkg().Path() == "testing" && obj.Name() == "T" } -func transformLink(args []string) ([]string, error) { +func transformLink(args []string) ([]string, func() error, error) { // We can't split by the ".a" extension, because cached object files // lack any extension. flags, paths := splitFlagsFromArgs(args) if err := fillBuildInfo(flags); err != nil { - return nil, err + return nil, nil, err } - tempDir, err := ioutil.TempDir("", "garble-build") - if err != nil { - return nil, err - } - deferred = append(deferred, func() error { - return os.RemoveAll(tempDir) - }) - // there should only ever be one archive/object file passed to the linker, // the file for the main package or entrypoint if len(paths) != 1 { - return nil, fmt.Errorf("expected exactly one link argument") + return nil, nil, fmt.Errorf("expected exactly one link argument") } importMap := func(importPath string) (objectPath string) { return buildInfo.imports[importPath].packagefile } - garbledObj, garbledImports, privateNameMap, err := obfuscateImports(paths[0], tempDir, importMap) + garbledObj, garbledImports, privateNameMap, err := obfuscateImports(paths[0], importMap) if err != nil { - return nil, err + return nil, nil, err } // Make sure -X works with garbled identifiers. To cover both garbled @@ -1291,7 +1282,7 @@ func transformLink(args []string) ([]string, error) { // Strip debug information and symbol tables. flags = append(flags, "-w", "-s") - return append(flags, garbledObj), nil + return append(flags, garbledObj), nil, nil } func splitFlagsFromArgs(all []string) (flags, args []string) { diff --git a/shared.go b/shared.go index d8d4b97..1ad0e6d 100644 --- a/shared.go +++ b/shared.go @@ -28,7 +28,7 @@ var cache *shared // loadShared the shared data passed from the entry garble process func loadShared() error { if cache == nil { - f, err := os.Open(os.Getenv("GARBLE_SHARED")) + f, err := os.Open(filepath.Join(sharedTempDir, "main-cache.gob")) if err != nil { return fmt.Errorf(`cannot open shared file, this is most likely due to not running "garble [command]"`) } @@ -41,23 +41,25 @@ func loadShared() error { return nil } -// saveShared the shared data to a file in order for subsequent -// garble processes to have access to the same data +// saveShared creates a temporary directory to share between garble processes. +// This directory also includes the gob-encoded cache global. func saveShared() (string, error) { - f, err := ioutil.TempFile("", "garble-shared") + dir, err := ioutil.TempDir("", "garble-shared") if err != nil { return "", err } + sharedCache := filepath.Join(dir, "main-cache.gob") + f, err := os.Create(sharedCache) + if err != nil { + return "", err + } defer f.Close() if err := gob.NewEncoder(f).Encode(&cache); err != nil { return "", err } - - os.Setenv("GARBLE_SHARED", f.Name()) - - return f.Name(), nil + return dir, nil } // options are derived from the flags