diff --git a/main.go b/main.go
index 459e779..7f92f93 100644
--- a/main.go
+++ b/main.go
@@ -97,13 +97,13 @@ var (
 	// origImporter is a go/types importer which uses the original versions
 	// of packages, without any obfuscation. This is helpful to make
 	// decisions on how to obfuscate our input code.
-	origImporter = importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) {
+	origImporter = importerWithMap(importer.ForCompiler(fset, "gc", func(path string) (io.ReadCloser, error) {
 		pkg, err := listPackage(path)
 		if err != nil {
 			return nil, err
 		}
 		return os.Open(pkg.Export)
-	})
+	}).(types.ImporterFrom).ImportFrom)
 
 	// Basic information about the package being currently compiled or linked.
 	curPkg *listedPackage
@@ -119,6 +119,19 @@ var (
 	opts *flagOptions
 )
 
+type importerWithMap func(path, dir string, mode types.ImportMode) (*types.Package, error)
+
+func (fn importerWithMap) Import(path string) (*types.Package, error) {
+	panic("should never be called")
+}
+
+func (fn importerWithMap) ImportFrom(path, dir string, mode types.ImportMode) (*types.Package, error) {
+	if path2 := curPkg.ImportMap[path]; path2 != "" {
+		path = path2
+	}
+	return fn(path, dir, mode)
+}
+
 func obfuscatedTypesPackage(path string) *types.Package {
 	entry, ok := importCfgEntries[path]
 	if !ok {
@@ -585,20 +598,6 @@ func transformCompile(args []string) ([]string, error) {
 		},
 	}
 
-	// The standard library vendors external packages, which results in them
-	// listing "golang.org/x/foo" in go list -json's Deps, plus an ImportMap
-	// entry to remap them to "vendor/golang.org/x/foo".
-	// We support that edge case in listPackage, presumably, though it seems
-	// like importer.ForCompiler with a lookup function isn't capable of it.
-	// It does work without an explicit lookup func though, which results in
-	// extra calls to 'go list'.
-	// Since this is a rare edge case and only occurs for a few std
-	// packages, do the extra 'go list' calls for now.
-	// TODO(mvdan): remove once https://github.com/golang/go/issues/44630 is fixed
-	if curPkg.Standard && len(curPkg.ImportMap) > 0 {
-		origImporter = importer.Default()
-	}
-
 	origTypesConfig := types.Config{Importer: origImporter}
 	tf.pkg, err = origTypesConfig.Check(curPkg.ImportPath, fset, files, tf.info)
 	if err != nil {
diff --git a/testdata/scripts/crossbuild.txt b/testdata/scripts/crossbuild.txt
new file mode 100644
index 0000000..f2c722b
--- /dev/null
+++ b/testdata/scripts/crossbuild.txt
@@ -0,0 +1,26 @@
+# TODO: always cross-build, even on a windows/arm host
+env GOOS=windows
+env GOARCH=arm
+
+env GOPRIVATE='*'
+
+# TODO: seems like the linker fails for some reason with the full binary build below.
+# For now, just test that we can build the library.
+garble build net/http
+
+# Link a binary importing net/http, which will catch whether or not we
+# support ImportMap when linking.
+# garble build
+
+-- go.mod --
+module test/main
+
+go 1.16
+-- main.go --
+package main
+
+import "net/http"
+
+func main() {
+	http.ListenAndServe("", nil)
+}