libchrome: downgrade mojom types at arc build time

This is to prepare for synchronizing .mojom files with those in
chromium/src which are in new syntax.

Add the mojom downgrading python script from crrev.com/c/2082513.
In soong/bindings_generator.go, define new ModuleFactory executing it
and register as new Module generate_mojom_downgraded_files.

The change will be cherry-picked to ARC branches to use in Android.bp:
downgraded .mojom files are generated from the original .mojom files
using the new module, then they would (replace the original .mojom files
to) be used as input to generate bindings.
Files in old syntax will be output as is.

Note that previously, when generating the bindings, it is assumed that
the input .mojom files are located under the module's local source
directory, which is then given as value of argument depth to
mojo/public/tools/bindings/mojom_bindings_generator.py.
The mojom_bindings_generator.py computes the relative path (hence
output path) accordingly.
Since this is no longer true, the source root would now be computed per
input .mojom file.

Bug: 149070836, chromium:1035484
Test: local build; compare downgraded mojom files and generated bindings

Change-Id: I86a6c67c5ae7056f3c9f617a1c5a9c954c10aa0d

Author: hscham

mojo/public/tools/bindings/mojom_types_downgrader.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""Downgrades *.mojom files to the old mojo types for remotes and receivers."""
+
+import argparse
+import fnmatch
+import os
+import re
+import shutil
+import sys
+import tempfile
+
+# List of patterns and replacements to match and use against the contents of a
+# mojo file. Each replacement string will be used with Python string's format()
+# function, so the '{}' substring is used to mark where the mojo type should go.
+_MOJO_REPLACEMENTS = {
+    r'pending_remote': r'{}',
+    r'pending_receiver': r'{}&',
+    r'pending_associated_remote': r'associated {}',
+    r'pending_associated_receiver': r'associated {}&',
+}
+
+# Pre-compiled regular expression that matches against any of the replacements.
+_REGEXP_PATTERN = re.compile(
+    r'|'.join(
+        ['{}\s*<\s*(.*?)\s*>'.format(k) for k in _MOJO_REPLACEMENTS.keys()]),
+    flags=re.DOTALL)
+
+
+def ReplaceFunction(match_object):
+  """Returns the right replacement for the string matched against the regexp."""
+  for index, (match, repl) in enumerate(_MOJO_REPLACEMENTS.items(), 1):
+    if match_object.group(0).startswith(match):
+      return repl.format(match_object.group(index))
+
+
+def DowngradeFile(path, output_dir=None):
+  """Downgrades the mojom file specified by |path| to the old mojo types.
+
+  Optionally pass |output_dir| to place the result under a separate output
+  directory, preserving the relative path to the file included in |path|.
+  """
+  # Use a temporary file to dump the new contents after replacing the patterns.
+  with open(path) as src_mojo_file:
+    with tempfile.NamedTemporaryFile(mode='w', delete=False) as tmp_mojo_file:
+      tmp_contents = _REGEXP_PATTERN.sub(ReplaceFunction, src_mojo_file.read())
+      tmp_mojo_file.write(tmp_contents)
+
+  # Files should be placed in the desired output directory
+  if output_dir:
+    output_filepath = os.path.join(output_dir, os.path.basename(path))
+    if not os.path.exists(output_dir):
+      os.makedirs(output_dir)
+  else:
+    output_filepath = path
+
+  # Write the new contents preserving the original file's attributes.
+  shutil.copystat(path, tmp_mojo_file.name)
+  shutil.move(tmp_mojo_file.name, output_filepath)
+
+  # Make sure to "touch" the new file so that access, modify and change times
+  # are always newer than the source file's, otherwise Modify time will be kept
+  # as per the call to shutil.copystat(), causing unnecessary generations of the
+  # output file in subsequent builds due to ninja considering it dirty.
+  os.utime(output_filepath, None)
+
+
+def DowngradeDirectory(path, output_dir=None):
+  """Downgrades mojom files inside directory |path| to the old mojo types.
+
+  Optionally pass |output_dir| to place the result under a separate output
+  directory, preserving the relative path to the file included in |path|.
+  """
+  # We don't have recursive glob.glob() nor pathlib.Path.rglob() in Python 2.7
+  mojom_filepaths = []
+  for dir_path, _, filenames in os.walk(path):
+    for filename in fnmatch.filter(filenames, "*mojom"):
+      mojom_filepaths.append(os.path.join(dir_path, filename))
+
+  for path in mojom_filepaths:
+    absolute_dirpath = os.path.dirname(os.path.abspath(path))
+    if output_dir:
+      dest_dirpath = output_dir + absolute_dirpath
+    else:
+      dest_dirpath = absolute_dirpath
+    DowngradeFile(path, dest_dirpath)
+
+
+def DowngradePath(src_path, output_dir=None):
+  """Downgrades the mojom files pointed by |src_path| to the old mojo types.
+
+  Optionally pass |output_dir| to place the result under a separate output
+  directory, preserving the relative path to the file included in |path|.
+  """
+  if os.path.isdir(src_path):
+    DowngradeDirectory(src_path, output_dir)
+  elif os.path.isfile(src_path):
+    DowngradeFile(src_path, output_dir)
+  else:
+    print(">>> {} not pointing to a valid file or directory".format(src_path))
+    sys.exit(1)
+
+
+def main():
+  parser = argparse.ArgumentParser(
+      description="Downgrade *.mojom files to use the old mojo types.")
+  parser.add_argument(
+      "srcpath", help="path to the file or directory to apply the conversion")
+  parser.add_argument(
+      "--outdir", help="the directory to place the converted file(s) under")
+  args = parser.parse_args()
+
+  DowngradePath(args.srcpath, args.outdir)
+
+
+if __name__ == "__main__":
+  sys.exit(main())

soong/bindings_generator.go

@@ -12,6 +12,7 @@ import (
 )
 
 func init() {
+	android.RegisterModuleType("generate_mojom_downgraded_files", mojomDowngradedFilesFactory)
 	android.RegisterModuleType("generate_mojom_pickles", mojomPicklesFactory)
 	android.RegisterModuleType("generate_mojom_headers", mojomHeadersFactory)
 	android.RegisterModuleType("generate_mojom_srcs", mojomSrcsFactory)
@@ -22,8 +23,19 @@ var (
 	pctx = android.NewPackageContext("android/soong/external/libchrome")
 
 	mojomBindingsGenerator = pctx.HostBinToolVariable("mojomBindingsGenerator", "mojom_bindings_generator")
+	mojomTypesDowngrader   = pctx.HostBinToolVariable("mojomTypesDowngrader", "mojom_types_downgrader")
 	mergeZips              = pctx.HostBinToolVariable("mergeZips", "merge_zips")
 
+	downgradeMojomTypesRule = pctx.StaticRule("downgradeMojomTypesRule", blueprint.RuleParams{
+		Command: `${mojomTypesDowngrader}
+		${in}
+		--outdir ${outDir}`,
+		CommandDeps: []string{
+			"${mojomTypesDowngrader}",
+		},
+		Description: "Downgrade mojom files $in => $out",
+	}, "outDir")
+
 	generateMojomPicklesRule = pctx.StaticRule("generateMojomPicklesRule", blueprint.RuleParams{
 		Command: `${mojomBindingsGenerator}
 		--use_bundled_pylibs parse
@@ -65,6 +77,72 @@ var (
 	})
 )
 
+type mojomDowngradedFilesProperties struct {
+	// list of input files
+	Srcs []string
+}
+
+type mojomDowngradedFiles struct {
+	android.ModuleBase
+
+	properties mojomDowngradedFilesProperties
+
+	generatedSrcs android.Paths
+	outDir        android.Path
+}
+
+var _ genrule.SourceFileGenerator = (*mojomDowngradedFiles)(nil)
+
+func (m *mojomDowngradedFiles) DepsMutator(ctx android.BottomUpMutatorContext) {
+	android.ExtractSourcesDeps(ctx, m.properties.Srcs)
+}
+
+func (m *mojomDowngradedFiles) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+	m.outDir = android.PathForModuleGen(ctx, "")
+
+	for _, in := range ctx.ExpandSources(m.properties.Srcs, nil) {
+		if !strings.HasSuffix(in.Rel(), ".mojom") {
+			ctx.PropertyErrorf("srcs", "Source is not a .mojom file: %s", in.Rel())
+			continue
+		}
+
+		out := android.PathForModuleGen(ctx, in.Rel())
+		m.generatedSrcs = append(m.generatedSrcs, out)
+
+		ctx.ModuleBuild(pctx, android.ModuleBuildParams{
+			Rule:   downgradeMojomTypesRule,
+			Input:  in,
+			Output: out,
+			Args: map[string]string{
+				"outDir":  path.Dir(out.String()),
+			},
+		})
+	}
+}
+
+func (m *mojomDowngradedFiles) GeneratedHeaderDirs() android.Paths {
+	return nil
+}
+
+func (m *mojomDowngradedFiles) GeneratedDeps() android.Paths {
+	return append(android.Paths{}, m.generatedSrcs...)
+}
+
+func (m *mojomDowngradedFiles) GeneratedSourceFiles() android.Paths {
+	return append(android.Paths{}, m.generatedSrcs...)
+}
+
+func (m *mojomDowngradedFiles) Srcs() android.Paths {
+	return append(android.Paths{}, m.generatedSrcs...)
+}
+
+func mojomDowngradedFilesFactory() android.Module {
+	m := &mojomDowngradedFiles{}
+	m.AddProperties(&m.properties)
+	android.InitAndroidModule(m)
+	return m
+}
+
 type mojomPicklesProperties struct {
 	// list of input files
 	Srcs []string
@@ -88,13 +166,14 @@ func (m *mojomPickles) DepsMutator(ctx android.BottomUpMutatorContext) {
 func (m *mojomPickles) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 	m.outDir = android.PathForModuleGen(ctx, "")
 
-	packagePath := android.PathForModuleSrc(ctx, "")
-
 	for _, in := range ctx.ExpandSources(m.properties.Srcs, nil) {
 		if !strings.HasSuffix(in.Rel(), ".mojom") {
 			ctx.PropertyErrorf("srcs", "Source is not a .mojom file: %s", in.Rel())
 			continue
 		}
+
+		srcRoot := strings.TrimSuffix(in.String(), in.Rel())
+
 		relStem := strings.TrimSuffix(in.Rel(), ".mojom")
 
 		out := android.PathForModuleGen(ctx, relStem+".p")
@@ -105,9 +184,8 @@ func (m *mojomPickles) GenerateAndroidBuildActions(ctx android.ModuleContext) {
 			Input:  in,
 			Output: out,
 			Args: map[string]string{
-				"package": packagePath.Rel(),
+				"package": srcRoot,
 				"outDir":  m.outDir.String(),
-				"flags":   fmt.Sprintf("-I=%s:%s", packagePath, packagePath),
 			},
 		})
 	}
@@ -237,7 +315,6 @@ func (p *mojomGenerationProperties) generateBuildActions(
 	mojomGenerator string,
 	descriptions []mojomSrcsRuleDescription,
 ) android.Paths {
-	packageName := android.PathForModuleSrc(ctx, "").Rel()
 	outDir := android.PathForModuleGen(ctx, "")
 	implicitDeps := p.implicitDeps(ctx)
 	templateDir := p.templateDir(ctx)
@@ -249,6 +326,7 @@ func (p *mojomGenerationProperties) generateBuildActions(
 			continue
 		}
 		relStem := strings.TrimSuffix(in.Rel(), ".mojom")
+		srcRoot := strings.TrimSuffix(in.String(), in.Rel())
 
 		for _, description := range descriptions {
 			outs := android.WritablePaths{}
@@ -264,7 +342,7 @@ func (p *mojomGenerationProperties) generateBuildActions(
 				Outputs:   outs,
 				Args: map[string]string{
 					"mojomGenerator": mojomGenerator,
-					"package":        packageName,
+					"package":        srcRoot,
 					"flags":          fmt.Sprintf("%s %s", p.flags(ctx), description.extraFlags),
 					"outDir":         outDir.String(),
 					"templateDir":    templateDir,