Bindings generator spends a lot of time launching Perl and re-parsing the same files needlessly https://bugs.webkit.org/show_bug.cgi?id=310049 rdar://172693536 Reviewed by Darin Adler, Megan Gardner, and Abrar Rahman Protyasha. There is a significant amount of clean build overhead when generating derived sources, from launching a Perl instance per IDL file. It turns out that the CMake build uses a wrapper which first determines the set of stale IDLs, then shards them between a small number of Perl instances. Adopt this wrapper in the Xcode build, and make a few adjustments to it to make that work: - Make it possible to exclude generation for given hand-written implementations (EventListener) - Default the number of shards to the number of CPUs, instead of 1 - Make the preprocess-idls step optional, because the Xcode build does it separately Also, drive-by hoist shared work (parsing supplemental dependency files) out of the inner loop, which makes the win from sharing a process even bigger. This cuts a full minute (almost 5%) off clean builds for me, and has no measurable impact on incremental builds that touch IDL files. * Source/WebCore/DerivedSources.make: * Source/WebCore/bindings/scripts/generate-bindings-all.pl: * Source/WebCore/bindings/scripts/generate-bindings.pl: (generateBindings): Canonical link: https://commits.webkit.org/309433@main
diff --git a/Source/WebCore/DerivedSources.make b/Source/WebCore/DerivedSources.make index 40b797d6..aa2dd83 100644 --- a/Source/WebCore/DerivedSources.make +++ b/Source/WebCore/DerivedSources.make
@@ -2683,30 +2683,24 @@ vpath %.idl $(ADDITIONAL_BINDING_IDLS_PATHS) $(WebCore)/bindings/scripts # ------------------------------------------------- -define GENERATE_BINDINGS_template +JS_DOM_HEADERS_PATTERNS = $(subst .h,%h,$(JS_DOM_HEADERS)) +JS_DOM_IMPLEMENTATIONS_PATTERNS = $(subst .cpp,%cpp,$(JS_DOM_IMPLEMENTATIONS)) -JS$(call get_bare_name,$(1)).cpp JS$(call get_bare_name,$(1)).h: $(1) $$(JS_BINDINGS_SCRIPTS) $$(IDL_ATTRIBUTES_FILE) $$(IDL_INTERMEDIATE_FILES) $$(FEATURE_AND_PLATFORM_FLAGS_RESPONSE_FILE) | $(IDL_FILE_NAMES_LIST) - $$(PERL) $$(WebCore)/bindings/scripts/generate-bindings.pl \ - $$(IDL_COMMON_ARGS) \ - --defines "$$(FEATURE_AND_PLATFORM_DEFINES) LANGUAGE_JAVASCRIPT" \ +$(JS_DOM_HEADERS_PATTERNS) $(JS_DOM_IMPLEMENTATIONS_PATTERNS): $(JS_BINDING_IDLS) $(JS_BINDINGS_SCRIPTS) \ + $(IDL_ATTRIBUTES_FILE) $(IDL_INTERMEDIATE_FILES) \ + $(FEATURE_AND_PLATFORM_FLAGS_RESPONSE_FILE) \ + | $(IDL_FILE_NAMES_LIST) + $(PERL) $(WebCore)/bindings/scripts/generate-bindings-all.pl \ + --outputDir . \ + --idlFilesList $(IDL_FILE_NAMES_LIST) \ + --supplementalDependencyFile $(SUPPLEMENTAL_DEPENDENCY_FILE) \ + --idlAttributesFile $(IDL_ATTRIBUTES_FILE) \ + --defines "$(FEATURE_AND_PLATFORM_DEFINES) LANGUAGE_JAVASCRIPT" \ --generator JS \ - --idlAttributesFile $$(IDL_ATTRIBUTES_FILE) \ - --idlFileNamesList $(IDL_FILE_NAMES_LIST) \ - --supplementalDependencyFile $$(SUPPLEMENTAL_DEPENDENCY_FILE) \ - $$< -endef + $(addprefix --generatorDependency ,$(JS_BINDINGS_SCRIPTS)) \ + --exclude EventListener.idl # ------------------------------------------------- -$(foreach IDL_FILE,$(JS_BINDING_IDLS),$(eval $(call GENERATE_BINDINGS_template,$(IDL_FILE)))) - -ifneq ($(NO_SUPPLEMENTAL_FILES),1) --include $(SUPPLEMENTAL_MAKEFILE_DEPS) -endif - -ifneq ($(NO_SUPPLEMENTAL_FILES),1) --include $(JS_DOM_HEADERS:.h=.dep) -endif - # WebCore JS Builtins WebCore_BUILTINS_SOURCES = \
diff --git a/Source/WebCore/bindings/scripts/generate-bindings-all.pl b/Source/WebCore/bindings/scripts/generate-bindings-all.pl index adcf865..ccec7dc 100755 --- a/Source/WebCore/bindings/scripts/generate-bindings-all.pl +++ b/Source/WebCore/bindings/scripts/generate-bindings-all.pl
@@ -45,9 +45,10 @@ my $supplementalDependencyFile; my @ppExtraOutput; my @ppExtraArgs; -my $numOfJobs = 1; +my $numOfJobs; my $idlAttributesFile; my $showProgress; +my @exclude; GetOptions('outputDir=s' => \$outputDirectory, 'idlFilesList=s' => \$idlFilesList, @@ -62,34 +63,51 @@ 'ppExtraArgs=s@' => \@ppExtraArgs, 'idlAttributesFile=s' => \$idlAttributesFile, 'numOfJobs=i' => \$numOfJobs, + 'exclude=s@' => \@exclude, 'showProgress' => \$showProgress); +if (!defined $numOfJobs) { + $numOfJobs = `sysctl -n hw.activecpu 2>/dev/null` || `nproc 2>/dev/null` || 4; + chomp $numOfJobs; +} + +$idlFileNamesList = $idlFilesList if !defined $idlFileNamesList; + $| = 1; my @idlFiles; open(my $fh, '<', $idlFilesList) or die "Cannot open $idlFilesList"; @idlFiles = map { CygwinPathIfNeeded(s/\r?\n?$//r) } <$fh>; close($fh) or die; +if (@exclude) { + my %excluded = map { $_ => 1 } @exclude; + @idlFiles = grep { !$excluded{basename($_)} } @idlFiles; +} + my @ppIDLFiles; -open($fh, '<', $ppIDLFilesList) or die "Cannot open $ppIDLFilesList"; -@ppIDLFiles = map { CygwinPathIfNeeded(s/\r?\n?$//r) } <$fh>; -close($fh) or die; +if ($ppIDLFilesList) { + open($fh, '<', $ppIDLFilesList) or die "Cannot open $ppIDLFilesList"; + @ppIDLFiles = map { CygwinPathIfNeeded(s/\r?\n?$//r) } <$fh>; + close($fh) or die; +} my %oldSupplements; my %newSupplements; if ($supplementalDependencyFile) { - my @output = ($supplementalDependencyFile, @ppExtraOutput); - my @deps = ($ppIDLFilesList, @ppIDLFiles, @generatorDependency); - if (needsUpdate(\@output, \@deps)) { - readSupplementalDependencyFile($supplementalDependencyFile, \%oldSupplements) if -e $supplementalDependencyFile; - my @args = (File::Spec->catfile($scriptDir, 'preprocess-idls.pl'), - '--defines', $defines, - '--idlFileNamesList', $ppIDLFilesList, - '--supplementalDependencyFile', $supplementalDependencyFile, - '--idlAttributesFile', $idlAttributesFile, - @ppExtraArgs); - printProgress("Preprocess IDL"); - executeCommand($perl, @args) == 0 or die; + if ($ppIDLFilesList) { + my @output = ($supplementalDependencyFile, @ppExtraOutput); + my @deps = ($ppIDLFilesList, @ppIDLFiles, @generatorDependency); + if (needsUpdate(\@output, \@deps)) { + readSupplementalDependencyFile($supplementalDependencyFile, \%oldSupplements) if -e $supplementalDependencyFile; + my @args = (File::Spec->catfile($scriptDir, 'preprocess-idls.pl'), + '--defines', $defines, + '--idlFileNamesList', $ppIDLFilesList, + '--supplementalDependencyFile', $supplementalDependencyFile, + '--idlAttributesFile', $idlAttributesFile, + @ppExtraArgs); + printProgress("Preprocess IDL"); + executeCommand($perl, @args) == 0 or die; + } } readSupplementalDependencyFile($supplementalDependencyFile, \%newSupplements); } @@ -98,10 +116,10 @@ '--defines', $defines, '--generator', $generator, '--outputDir', $outputDirectory, - '--preprocessor', $preprocessor, '--idlAttributesFile', $idlAttributesFile, '--idlFileNamesList', $idlFileNamesList, '--write-dependencies'); +push @args, '--preprocessor', $preprocessor if $preprocessor; push @args, '--supplementalDependencyFile', $supplementalDependencyFile if $supplementalDependencyFile; my %directoryCache;
diff --git a/Source/WebCore/bindings/scripts/generate-bindings.pl b/Source/WebCore/bindings/scripts/generate-bindings.pl index 4c4b848..d820a5f 100755 --- a/Source/WebCore/bindings/scripts/generate-bindings.pl +++ b/Source/WebCore/bindings/scripts/generate-bindings.pl
@@ -79,6 +79,38 @@ $outputHeadersDirectory = $outputDirectory; } +# Parse supplemental dependencies and IDL attributes once, shared across all files. +my %supplementalDependencies; +if ($supplementalDependencyFile) { + # The format of a supplemental dependency file: + # + # DOMWindow.idl P.idl Q.idl R.idl + # Document.idl S.idl + # Event.idl + # ... + # + # The above indicates that DOMWindow.idl is supplemented by P.idl, Q.idl and R.idl, + # Document.idl is supplemented by S.idl, and Event.idl is supplemented by no IDLs. + open FH, "< $supplementalDependencyFile" or die "Cannot open $supplementalDependencyFile\n"; + while (my $line = <FH>) { + my ($idlFile, @followingIdlFiles) = split(/\s+/, $line); + $supplementalDependencies{fileparse($idlFile)} = [sort @followingIdlFiles] if $idlFile; + } + close FH; +} + +my $idlAttributes; +{ + local $INPUT_RECORD_SEPARATOR; + open(JSON, "<", $idlAttributesFile) or die "Couldn't open $idlAttributesFile: $!"; + my $input = <JSON>; + close(JSON); + + my $jsonDecoder = JSON::PP->new->utf8; + my $jsonHashRef = $jsonDecoder->decode($input); + $idlAttributes = $jsonHashRef->{attributes}; +} + generateBindings($_) for (@ARGV); sub generateBindings @@ -91,38 +123,6 @@ } my $targetInterfaceName = fileparse($targetIdlFile, ".idl"); - my $idlFound = 0; - my %supplementalDependencies; - if ($supplementalDependencyFile) { - # The format of a supplemental dependency file: - # - # DOMWindow.idl P.idl Q.idl R.idl - # Document.idl S.idl - # Event.idl - # ... - # - # The above indicates that DOMWindow.idl is supplemented by P.idl, Q.idl and R.idl, - # Document.idl is supplemented by S.idl, and Event.idl is supplemented by no IDLs. - open FH, "< $supplementalDependencyFile" or die "Cannot open $supplementalDependencyFile\n"; - while (my $line = <FH>) { - my ($idlFile, @followingIdlFiles) = split(/\s+/, $line); - $supplementalDependencies{fileparse($idlFile)} = [sort @followingIdlFiles] if $idlFile; - } - close FH; - } - - my $input; - { - local $INPUT_RECORD_SEPARATOR; - open(JSON, "<", $idlAttributesFile) or die "Couldn't open $idlAttributesFile: $!"; - $input = <JSON>; - close(JSON); - } - - my $jsonDecoder = JSON::PP->new->utf8; - my $jsonHashRef = $jsonDecoder->decode($input); - my $idlAttributes = $jsonHashRef->{attributes}; - # Parse the target IDL file. my $targetParser = IDLParser->new(!$verbose); my $targetDocument = $targetParser->Parse($targetIdlFile, $defines, $preprocessor, $idlAttributes);