From 961daf20c434173472ed02789a77622b250be972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Fri, 19 Mar 2021 17:55:29 +0100 Subject: [PATCH] rework the position obfuscator (#282) First, rename line_obfuscator.go to position.go. We obfuscate filenames, not just line numbers, and "obfuscator" is a bit redundant. Second, use "/*line :x*/" comments rather than the "//line :x" form, as the former allows us to insert them in any position without adding unnecessary newlines. This will be important for changing the position of call sites, which will be important for "garble reverse". Third, do not rely on go/ast to remove and add comments. Since they are free-floating, we can very easily end up with misplaced comments, especially as the literal obfuscator heavily modifies the AST. The new method prints and re-parses the file, to ensure all node positions are consistent with a buffer, buf1. Then, we copy the contents into a new buffer, buf2, while inserting the comments that we need. The new method also modifies line numbers at the very end of obfuscating a Go file, instead of at the very beginning. That's going to be more robust long-term, as we will also obfuscate line numbers for any additions or modifications to the AST. Fourth, detachedDirectives is unnecessary, as we can accomplish the same with two simple prefix matches. Finally, this means we can stop using detachedComments entirely, as printFile already inserts the comments we need. For #5. --- line_obfuscator.go | 137 ------------------------------------------- main.go | 143 ++++++++++++++++++++------------------------- position.go | 119 +++++++++++++++++++++++++++++++++++++ 3 files changed, 182 insertions(+), 217 deletions(-) delete mode 100644 line_obfuscator.go create mode 100644 position.go diff --git a/line_obfuscator.go b/line_obfuscator.go deleted file mode 100644 index 027a9cf..0000000 --- a/line_obfuscator.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright (c) 2020, The Garble Authors. -// See LICENSE for licensing information. - -package main - -import ( - "fmt" - "go/ast" - "strings" -) - -// PosMin is the smallest correct value for the line number. -// Source: https://go.googlesource.com/go/+/refs/heads/master/src/cmd/compile/internal/syntax/parser_test.go#229 -const PosMin = 1 - -// detachedDirectives is a list of Go compiler directives which don't need to go -// right next to a Go declaration. Unlike all other detached comments, these -// need to be kept around as they alter compiler behavior. -var detachedDirectives = []string{ - "// +build", - "//go:linkname", - "//go:cgo_ldflag", - "//go:cgo_dynamic_linker", - "//go:cgo_export_static", - "//go:cgo_export_dynamic", - "//go:cgo_import_static", - "//go:cgo_import_dynamic", -} - -func isDirective(text string, directives []string) bool { - for _, prefix := range directives { - if strings.HasPrefix(text, prefix) { - return true - } - } - return false -} - -func prependComment(group *ast.CommentGroup, comment *ast.Comment) *ast.CommentGroup { - if group == nil { - return &ast.CommentGroup{List: []*ast.Comment{comment}} - } - - group.List = append([]*ast.Comment{comment}, group.List...) - return group -} - -// Remove all comments from CommentGroup except //go: directives. -// go:linkname directives are removed, since they're collected and rewritten -// separately. -func clearCommentGroup(group *ast.CommentGroup) *ast.CommentGroup { - if group == nil { - return nil - } - - var comments []*ast.Comment - for _, comment := range group.List { - if strings.HasPrefix(comment.Text, "//go:") && !strings.HasPrefix(comment.Text, "//go:linkname") { - comments = append(comments, &ast.Comment{Text: comment.Text}) - } - } - if len(comments) == 0 { - return nil - } - return &ast.CommentGroup{List: comments} -} - -// Remove all comments from Doc (if any) except //go: directives. -func clearNodeComments(node ast.Node) { - switch n := node.(type) { - case *ast.Field: - n.Doc = clearCommentGroup(n.Doc) - n.Comment = nil - case *ast.ImportSpec: - n.Doc = clearCommentGroup(n.Doc) - n.Comment = nil - case *ast.ValueSpec: - n.Doc = clearCommentGroup(n.Doc) - n.Comment = nil - case *ast.TypeSpec: - n.Doc = clearCommentGroup(n.Doc) - n.Comment = nil - case *ast.GenDecl: - n.Doc = clearCommentGroup(n.Doc) - case *ast.FuncDecl: - n.Doc = clearCommentGroup(n.Doc) - case *ast.File: - n.Doc = clearCommentGroup(n.Doc) - } -} - -// transformLineInfo removes the comment except go directives and build tags. Converts comments to the node view. -// It returns comments not attached to declarations and names of declarations which cannot be renamed. -func (tf *transformer) transformLineInfo(file *ast.File, filename string) (detachedComments []string, f *ast.File) { - prefix := "" - if strings.HasPrefix(filename, "_cgo_") { - prefix = "_cgo_" - } - - // Save build tags and add file name leak protection - for _, group := range file.Comments { - for _, comment := range group.List { - if isDirective(comment.Text, detachedDirectives) { - detachedComments = append(detachedComments, comment.Text) - } - } - } - detachedComments = append(detachedComments, "", "//line "+prefix+":1") - file.Comments = nil - - ast.Inspect(file, func(node ast.Node) bool { - clearNodeComments(node) - return true - }) - - for _, decl := range file.Decls { - var doc **ast.CommentGroup - switch decl := decl.(type) { - case *ast.FuncDecl: - doc = &decl.Doc - case *ast.GenDecl: - doc = &decl.Doc - } - newName := "" - if !opts.Tiny { - origPos := fmt.Sprintf("%s:%d", filename, fset.Position(decl.Pos()).Offset) - newName = hashWith(curPkg.GarbleActionID, origPos) + ".go" - // log.Printf("%q hashed with %x to %q", origPos, curPkg.GarbleActionID, newName) - } - newPos := fmt.Sprintf("%s%s:1", prefix, newName) - - comment := &ast.Comment{Text: "//line " + newPos} - *doc = prependComment(*doc, comment) - } - - return detachedComments, file -} diff --git a/main.go b/main.go index 69e3fec..c6d6dc7 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,6 @@ import ( "go/ast" "go/importer" "go/parser" - "go/printer" "go/token" "go/types" "io" @@ -85,8 +84,6 @@ var ( fset = token.NewFileSet() sharedTempDir = os.Getenv("GARBLE_SHARED") - printConfig = printer.Config{Mode: printer.RawFormat} - // origImporter is a go/types importer which uses the original versions // of packages, without any obfuscation. This is helpful to make // decisions on how to obfuscate our input code. @@ -541,6 +538,7 @@ func transformCompile(args []string) ([]string, error) { var files []*ast.File for _, path := range paths { + // Note that we want file, err := parser.ParseFile(fset, path, nil, parser.ParseComments) if err != nil { return nil, err @@ -597,17 +595,6 @@ func transformCompile(args []string) ([]string, error) { flags = flagSetValue(flags, "-trimpath", sharedTempDir+"=>;"+trimpath) // log.Println(flags) - detachedComments := make([][]string, len(files)) - - for i, file := range files { - name := filepath.Base(filepath.Clean(paths[i])) - - comments, file := tf.transformLineInfo(file, name) - tf.handleDirectives(comments) - - detachedComments[i], files[i] = comments, file - } - // If this is a package to obfuscate, swap the -p flag with the new // package path. newPkgPath := "" @@ -618,7 +605,9 @@ func transformCompile(args []string) ([]string, error) { newPaths := make([]string, 0, len(files)) for i, file := range files { - origName := filepath.Base(filepath.Clean(paths[i])) + tf.handleDirectives(file.Comments) + + origName := filepath.Base(paths[i]) name := origName switch { case curPkg.ImportPath == "runtime": @@ -678,12 +667,14 @@ func transformCompile(args []string) ([]string, error) { file.Name.Name = newPkgPath } + src, err := printFile(file) + if err != nil { + return nil, err + } + // Uncomment for some quick debugging. Do not delete. // if curPkg.Private { - // fmt.Fprintf(os.Stderr, "\n-- %s/%s --\n", curPkg.ImportPath, origName) - // if err := printConfig.Fprint(os.Stderr, fset, file); err != nil { - // return nil, err - // } + // fmt.Fprintf(os.Stderr, "\n-- %s/%s --\n%s", curPkg.ImportPath, origName, src) // } tempFile, err := os.CreateTemp(sharedTempDir, name+".*.go") @@ -691,13 +682,7 @@ func transformCompile(args []string) ([]string, error) { return nil, err } defer tempFile.Close() - - for _, comment := range detachedComments[i] { - if _, err := tempFile.Write([]byte(comment + "\n")); err != nil { - return nil, err - } - } - if err := printConfig.Fprint(tempFile, fset, file); err != nil { + if _, err := tempFile.Write(src); err != nil { return nil, err } if opts.DebugDir != "" { @@ -708,14 +693,7 @@ func transformCompile(args []string) ([]string, error) { } debugFilePath := filepath.Join(pkgDebugDir, origName) - debugFile, err := os.Create(debugFilePath) - if err != nil { - return nil, err - } - if err := printConfig.Fprint(debugFile, fset, file); err != nil { - return nil, err - } - if err := debugFile.Close(); err != nil { + if err := os.WriteFile(debugFilePath, src, 0666); err != nil { return nil, err } } @@ -736,56 +714,61 @@ func transformCompile(args []string) ([]string, error) { // // Right now, this means recording what local names are used with go:linkname, // and rewriting those directives to use obfuscated name from other packages. -func (tf *transformer) handleDirectives(comments []string) { - for i, comment := range comments { - if !strings.HasPrefix(comment, "//go:linkname ") { - continue - } - fields := strings.Fields(comment) - if len(fields) != 3 { - continue - } - // This directive has two arguments: "go:linkname localName newName" - localName := fields[1] +func (tf *transformer) handleDirectives(comments []*ast.CommentGroup) { + if !curPkg.Private { + return + } + for _, group := range comments { + for _, comment := range group.List { + if !strings.HasPrefix(comment.Text, "//go:linkname ") { + continue + } + fields := strings.Fields(comment.Text) + if len(fields) != 3 { + continue + } + // This directive has two arguments: "go:linkname localName newName" + localName := fields[1] - // The local name must not be obfuscated. - obj := tf.pkg.Scope().Lookup(localName) - if obj != nil { - tf.ignoreObjects[obj] = true - } + // The local name must not be obfuscated. + obj := tf.pkg.Scope().Lookup(localName) + if obj != nil { + tf.ignoreObjects[obj] = true + } - // If the new name is of the form "pkgpath.Name", and - // we've obfuscated "Name" in that package, rewrite the - // directive to use the obfuscated name. - target := strings.Split(fields[2], ".") - if len(target) != 2 { - continue - } - pkgPath, name := target[0], target[1] - if pkgPath == "runtime" && strings.HasPrefix(name, "cgo") { - continue // ignore cgo-generated linknames - } - lpkg, err := listPackage(pkgPath) - if err != nil { - continue // probably a made up symbol name - } - if !lpkg.Private { - continue // ignore non-private symbols - } - obfPkg := obfuscatedTypesPackage(pkgPath) - if obfPkg != nil && obfPkg.Scope().Lookup(name) != nil { - continue // the name exists and was not garbled - } + // If the new name is of the form "pkgpath.Name", and + // we've obfuscated "Name" in that package, rewrite the + // directive to use the obfuscated name. + target := strings.Split(fields[2], ".") + if len(target) != 2 { + continue + } + pkgPath, name := target[0], target[1] + if pkgPath == "runtime" && strings.HasPrefix(name, "cgo") { + continue // ignore cgo-generated linknames + } + lpkg, err := listPackage(pkgPath) + if err != nil { + continue // probably a made up symbol name + } + if !lpkg.Private { + continue // ignore non-private symbols + } + obfPkg := obfuscatedTypesPackage(pkgPath) + if obfPkg != nil && obfPkg.Scope().Lookup(name) != nil { + continue // the name exists and was not garbled + } - // The name exists and was obfuscated; replace the - // comment with the obfuscated name. - newName := hashWith(lpkg.GarbleActionID, name) - newPkgPath := pkgPath - if pkgPath != "main" { - newPkgPath = lpkg.obfuscatedImportPath() + // The name exists and was obfuscated; replace the + // comment with the obfuscated name. + newName := hashWith(lpkg.GarbleActionID, name) + newPkgPath := pkgPath + if pkgPath != "main" { + newPkgPath = lpkg.obfuscatedImportPath() + } + fields[2] = newPkgPath + "." + newName + comment.Text = strings.Join(fields, " ") } - fields[2] = newPkgPath + "." + newName - comments[i] = strings.Join(fields, " ") } } diff --git a/position.go b/position.go new file mode 100644 index 0000000..1796a02 --- /dev/null +++ b/position.go @@ -0,0 +1,119 @@ +// Copyright (c) 2020, The Garble Authors. +// See LICENSE for licensing information. + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "path/filepath" + "sort" + "strings" +) + +func isDirective(text string) bool { + return strings.HasPrefix(text, "//go:") || strings.HasPrefix(text, "// +build") +} + +// printFile prints a Go file to a buffer, while also removing non-directive +// comments and adding extra compiler directives to obfuscate position +// information. +func printFile(file *ast.File) ([]byte, error) { + printConfig := printer.Config{Mode: printer.RawFormat} + + var buf1 bytes.Buffer + if err := printConfig.Fprint(&buf1, fset, file); err != nil { + return nil, err + } + src := buf1.Bytes() + + if !curPkg.Private { + // TODO(mvdan): make transformCompile handle non-private + // packages like runtime earlier on, to remove these checks. + return src, nil + } + + filename := fset.Position(file.Pos()).Filename + if strings.HasPrefix(filepath.Base(filename), "_cgo_") { + // cgo-generated files don't need changed line numbers. + // Plus, the compiler can complain rather easily. + return src, nil + } + + // Many parts of garble, notably the literal obfuscator, modify the AST. + // Unfortunately, comments are free-floating in File.Comments, + // and those are the only source of truth that go/printer uses. + // So the positions of the comments in the given file are wrong. + // The only way we can get the final ones is to parse again. + file, err := parser.ParseFile(fset, filename, src, parser.ParseComments) + if err != nil { + return nil, err + } + + // Keep the compiler directives, and change position info. + type commentToAdd struct { + offset int + text string + } + var toAdd []commentToAdd + addComment := func(offset int, text string) { + toAdd = append(toAdd, commentToAdd{offset, text}) + } + addComment(0, "/*line :1*/") + for _, group := range file.Comments { + for _, comment := range group.List { + if isDirective(comment.Text) { + // TODO(mvdan): merge with the zeroing below + pos := fset.Position(comment.Pos()) + addComment(pos.Offset, comment.Text) + } + } + } + // Remove all existing comments by making them whitespace. + for _, group := range file.Comments { + for _, comment := range group.List { + start := fset.Position(comment.Pos()).Offset + end := fset.Position(comment.End()).Offset + for i := start; i < end; i++ { + src[i] = ' ' + } + } + } + + for _, decl := range file.Decls { + newName := "" + if !opts.Tiny { + origPos := fmt.Sprintf("%s:%d", filename, fset.Position(decl.Pos()).Offset) + newName = hashWith(curPkg.GarbleActionID, origPos) + ".go" + // log.Printf("%q hashed with %x to %q", origPos, curPkg.GarbleActionID, newName) + } + newPos := fmt.Sprintf("%s:1", newName) + pos := fset.Position(decl.Pos()) + + // We use the /*text*/ form, since we can use multiple of them + // on a single line, and they don't require extra newlines. + addComment(pos.Offset, "/*line "+newPos+"*/") + } + + // We add comments in order. + sort.Slice(toAdd, func(i, j int) bool { + return toAdd[i].offset < toAdd[j].offset + }) + + copied := 0 + var buf2 bytes.Buffer + for _, comment := range toAdd { + buf2.Write(src[copied:comment.offset]) + buf2.WriteString(comment.text) + if strings.HasPrefix(comment.text, "//") { + buf2.WriteByte('\n') + } + + copied = comment.offset + } + buf2.Write(src[copied:]) + return buf2.Bytes(), nil +}