feat: add ESM tools in gax (#1459)

This PR does the following:

1. Adds 3 babel plugins to help transform ESM code to CJS: one to transform `path.dirname(fileURLToPath(import.meta)` to `__dirname`, another to turn `isEsm` to false, and lastly a final one to turn proxyquire into esmock.
3. Adds an ESM-path in compileProtos (searches for source code one level deeper, and generates `protos.js` and `protos.cjs` in es6 and amd, respectively)

Author: ddelgrosso1

tools/package.json

@@ -32,11 +32,17 @@
     "google-proto-files": "^4.0.0",
     "protobufjs-cli": "1.1.2",
     "rimraf": "^5.0.1",
+    "@babel/core": "^7.22.5",
+    "@babel/traverse": "^7.22.5",
     "uglify-js": "^3.17.0",
     "walkdir": "^0.4.0"
   },
   "repository": "googleapis/gax-nodejs",
   "devDependencies": {
+    "@babel/cli": "^7.22.5",
+    "@babel/types": "^7.22.5",
+    "@types/babel__core": "^7.20.1",
+    "@types/babel__traverse": "^7.20.1",
     "@types/mocha": "^9.0.0",
     "@types/ncp": "^2.0.1",
     "@types/uglify-js": "^3.17.0",

tools/src/compileProtos.ts

@@ -163,6 +163,13 @@ function updateDtsTypes(dts: string, enums: Set<string>): string {
 }
 
 function fixJsFile(js: string): string {
+  // 0. fix protobufjs import: we don't want the libraries to
+  // depend on protobufjs, so we re-export it from google-gax
+  js = js.replace(
+    'import * as $protobuf from "protobufjs/minimal"',
+    'import {protobufMinimal  as $protobuf} from "google-gax/build/src/protobuf.js"'
+  );
+
   // 1. fix protobufjs require: we don't want the libraries to
   // depend on protobufjs, so we re-export it from google-gax
   js = js.replace(
@@ -211,13 +218,19 @@ function fixDtsFile(dts: string): string {
  * @param {string[]} protoJsonFiles List of JSON files to parse
  * @return {Promise<string[]>} Resolves to an array of proto files.
  */
-async function buildListOfProtos(protoJsonFiles: string[]): Promise<string[]> {
+async function buildListOfProtos(
+  protoJsonFiles: string[],
+  esm?: boolean
+): Promise<string[]> {
   const result: string[] = [];
   for (const file of protoJsonFiles) {
     const directory = path.dirname(file);
     const content = await readFile(file);
     const list = JSON.parse(content.toString()).map((filePath: string) =>
-      path.join(directory, normalizePath(filePath))
+      // If we're in ESM, we're going to be in a directory level below normal
+      esm
+        ? path.join(directory, '..', normalizePath(filePath))
+        : path.join(directory, normalizePath(filePath))
     );
     result.push(...list);
   }
@@ -236,7 +249,8 @@ async function buildListOfProtos(protoJsonFiles: string[]): Promise<string[]> {
 async function compileProtos(
   rootName: string,
   protos: string[],
-  skipJson = false
+  skipJson = false,
+  esm = false
 ): Promise<void> {
   if (!skipJson) {
     // generate protos.json file from proto list
@@ -261,7 +275,9 @@ async function compileProtos(
   }
 
   // generate protos/protos.js from protos.json
-  const jsOutput = path.join('protos', 'protos.js');
+  const jsOutput = esm
+    ? path.join('protos', 'protos.cjs')
+    : path.join('protos', 'protos.js');
   const pbjsArgs4js = [
     '-r',
     rootName,
@@ -281,9 +297,34 @@ async function compileProtos(
   jsResult = fixJsFile(jsResult);
   await writeFile(jsOutput, jsResult);
 
+  let jsOutputEsm;
+  if (esm) {
+    jsOutputEsm = path.join('protos', 'protos.js');
+    const pbjsArgs4jsEsm = [
+      '-r',
+      rootName,
+      '--target',
+      'static-module',
+      '-p',
+      'protos',
+      '-p',
+      gaxProtos,
+      '-o',
+      jsOutputEsm,
+      '-w',
+      'es6',
+    ];
+    pbjsArgs4jsEsm.push(...protos);
+    await pbjsMain(pbjsArgs4jsEsm);
+
+    let jsResult = (await readFile(jsOutputEsm)).toString();
+    jsResult = fixJsFile(jsResult);
+    await writeFile(jsOutputEsm, jsResult);
+  }
+
   // generate protos/protos.d.ts
   const tsOutput = path.join('protos', 'protos.d.ts');
-  const pbjsArgs4ts = [jsOutput, '-o', tsOutput];
+  const pbjsArgs4ts = [esm ? jsOutputEsm! : jsOutput, '-o', tsOutput];
   await pbtsMain(pbjsArgs4ts);
 
   let tsResult = (await readFile(tsOutput)).toString();
@@ -330,27 +371,38 @@ export async function generateRootName(directories: string[]): Promise<string> {
 export async function main(parameters: string[]): Promise<void> {
   const protoJsonFiles: string[] = [];
   let skipJson = false;
+  let esm = false;
   const directories: string[] = [];
   for (const parameter of parameters) {
     if (parameter === '--skip-json') {
       skipJson = true;
       continue;
     }
+    if (parameter === '--esm') {
+      esm = true;
+      continue;
+    }
     // it's not an option so it's a directory
     const directory = parameter;
     directories.push(directory);
     protoJsonFiles.push(...(await findProtoJsonFiles(directory)));
   }
   const rootName = await generateRootName(directories);
-  const protos = await buildListOfProtos(protoJsonFiles);
-  await compileProtos(rootName, protos, skipJson);
+  if (esm) {
+    const esmProtos = await buildListOfProtos(protoJsonFiles, esm);
+    await compileProtos(rootName, esmProtos, skipJson, esm);
+  }
+  const protos = await buildListOfProtos(protoJsonFiles, esm);
+  await compileProtos(rootName, protos, skipJson, esm);
 }
 
 /**
  * Shows the usage information.
  */
 function usage() {
-  console.log(`Usage: node ${process.argv[1]} [--skip-json] directory ...`);
+  console.log(
+    `Usage: node ${process.argv[1]} [--skip-json] [--esm] directory ...`
+  );
   console.log(
     `Finds all files matching ${PROTO_LIST_REGEX} in the given directories.`
   );

tools/src/replaceESMMockingLib.ts

@@ -0,0 +1,64 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {Visitor, types} from '@babel/core';
+
+export interface PluginOptions {
+  opts?: {
+    fromLibName?: string;
+    toLibName?: string;
+  };
+}
+
+export default function replaceESMMockingLib(): {
+  visitor: Visitor<PluginOptions>;
+} {
+  return {
+    visitor: {
+      ImportDeclaration(path, state) {
+        const opts = state.opts || {};
+        const fromLib = opts.fromLibName || 'esmock';
+        const toLib = opts.toLibName || 'proxyquire';
+        const {node} = path;
+
+        node.specifiers.forEach(spec => {
+          if (spec.local.name !== fromLib) {
+            return;
+          }
+          spec.local.name = spec.local.name.replace(fromLib, toLib);
+        });
+
+        if (node.source.value !== fromLib) {
+          return;
+        }
+        node.source.value = node.source.value.replace(fromLib, toLib);
+      },
+      CallExpression(path, state) {
+        const opts = state.opts || {};
+        const fromLib = opts.fromLibName || 'esmock';
+        const toLib = opts.toLibName || 'proxyquire';
+        const {node} = path;
+
+        if (types.isIdentifier(node.callee)) {
+          if (node.callee.name !== fromLib) {
+            return;
+          }
+
+          node.callee.name = toLib;
+          path.parentPath.replaceWith(types.expressionStatement(node));
+        }
+      },
+    },
+  };
+}

tools/src/replaceImportMetaUrl.ts

@@ -0,0 +1,58 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// eslint-disable-next-line node/no-extraneous-import
+import {smart} from '@babel/template';
+import {Statement} from '@babel/types';
+import {Visitor} from '@babel/core';
+
+export interface PluginOptions {
+  opts?: {
+    replacementValue?: string;
+  };
+}
+
+export default function replaceImportMetaUrl(): {
+  visitor: Visitor<PluginOptions>;
+} {
+  return {
+    visitor: {
+      CallExpression(path, state) {
+        const opts = state.opts || {};
+        const replacementValue = opts.replacementValue || '__dirname';
+        const {node} = path;
+        if (
+          node.callee.type === 'MemberExpression' &&
+          node.callee.property.type === 'Identifier' &&
+          node.callee.property.name === 'dirname' &&
+          node.arguments[0].type === 'CallExpression' &&
+          node.arguments[0].callee.type === 'Identifier' &&
+          node.arguments[0].callee.name === 'fileURLToPath' &&
+          node.arguments[0].arguments[0].type === 'MemberExpression' &&
+          node.arguments[0].arguments[0].object.type === 'MetaProperty' &&
+          node.arguments[0].arguments[0].object.meta.type === 'Identifier' &&
+          node.arguments[0].arguments[0].object.meta.name === 'import' &&
+          node.arguments[0].arguments[0].object.property.type ===
+            'Identifier' &&
+          node.arguments[0].arguments[0].object.property.name === 'meta' &&
+          node.arguments[0].arguments[0].property.type === 'Identifier' &&
+          node.arguments[0].arguments[0].property.name === 'url'
+        ) {
+          const replacement = smart.ast`${replacementValue}` as Statement;
+          path.replaceWith(replacement);
+        }
+      },
+    },
+  };
+}

tools/src/toggleESMFlagVariable.ts

@@ -0,0 +1,44 @@
+// Copyright 2023 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import {Visitor, types} from '@babel/core';
+
+export interface PluginOptions {
+  opts?: {
+    variableIdentifier?: string;
+    replacementValue?: boolean;
+  };
+}
+
+export default function toggleESMFlagVariable(): {
+  visitor: Visitor<PluginOptions>;
+} {
+  return {
+    visitor: {
+      VariableDeclarator(path, state) {
+        const opts = state.opts || {};
+        const variableIdentifier = opts.variableIdentifier || 'isEsm';
+        const replacementValue = opts.replacementValue || false;
+        const {node} = path;
+        const identifier = node.id as types.Identifier;
+        if (
+          identifier.name === variableIdentifier &&
+          node.init?.type === 'BooleanLiteral'
+        ) {
+          node.init.value = replacementValue;
+        }
+      },
+    },
+  };
+}