From d3e98192756159fd460d1547d9840e73b473de90 Mon Sep 17 00:00:00 2001 From: Gil Date: Wed, 23 May 2018 11:42:17 -0700 Subject: Add a test synchronization script (#1303) * Add a project sync script * Give an error if the configuration references a group that doesn't exist * Fix hard_assert_test reference * Run sync_project to sort all project elements --- scripts/check_copyright.sh | 2 +- scripts/sync_project.rb | 569 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 570 insertions(+), 1 deletion(-) create mode 100755 scripts/sync_project.rb (limited to 'scripts') diff --git a/scripts/check_copyright.sh b/scripts/check_copyright.sh index 5cd8a18..cc83e29 100755 --- a/scripts/check_copyright.sh +++ b/scripts/check_copyright.sh @@ -22,7 +22,7 @@ options=( ) git grep "${options[@]}" \ - -- '*.'{c,cc,h,js,m,mm,py,sh,swift} \ + -- '*.'{c,cc,h,js,m,mm,py,rb,sh,swift} \ ':(exclude)**/third_party/**' if [[ $? == 0 ]]; then echo "ERROR: Missing copyright notices in the files above. Please fix." diff --git a/scripts/sync_project.rb b/scripts/sync_project.rb new file mode 100755 index 0000000..e34ae31 --- /dev/null +++ b/scripts/sync_project.rb @@ -0,0 +1,569 @@ +#!/usr/bin/ruby + +# Copyright 2018 Google +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Syncs Xcode project folder and target structure with the filesystem. This +# script finds all files on the filesystem that match the patterns supplied +# below and changes the project to match what it found. +# +# Run this script after adding/removing tests to keep the project in sync. + +require 'pathname' + +# Note that xcodeproj 1.5.8 appears to be broken +# https://github.com/CocoaPods/Xcodeproj/issues/572 +gem 'xcodeproj', '!= 1.5.8' +require 'xcodeproj' + + +def main() + # Make all filenames relative to the project root. + Dir.chdir(File.join(File.dirname(__FILE__), '..')) + + sync_firestore() +end + + +def sync_firestore() + project = Xcodeproj::Project.open('Firestore/Example/Firestore.xcodeproj') + + # Enable warnings after opening the project to avoid the warnings in + # xcodeproj itself + $VERBOSE = true + + s = Syncer.new(project, Dir.pwd) + + # Files on the filesystem that should be ignored. + s.ignore_files = [ + 'CMakeLists.txt', + 'InfoPlist.strings', + '*.plist', + + # b/79496027 + 'Firestore/core/test/firebase/firestore/remote/serializer_test.cc', + ] + + # Folder groups in the Xcode project that contain tests. + s.test_groups = [ + 'Tests', + 'CoreTests', + 'SwiftTests', + ] + + s.target 'Firestore_Tests_iOS' do |t| + t.source_files = [ + 'Firestore/Example/Tests/**', + 'Firestore/core/test/**', + 'Firestore/third_party/Immutable/Tests/**', + ] + t.exclude_files = [ + # needs to be in project but not in target + 'Firestore/Example/Tests/Tests-Info.plist', + + # These files are integration tests, handled below + 'Firestore/Example/Tests/Integration/**', + ] + end + + s.target 'Firestore_IntegrationTests_iOS' do |t| + t.source_files = [ + 'Firestore/Example/Tests/Integration/**', + 'Firestore/Example/Tests/Util/FSTEventAccumulator.mm', + 'Firestore/Example/Tests/Util/FSTHelpers.mm', + 'Firestore/Example/Tests/Util/FSTIntegrationTestCase.mm', + 'Firestore/Example/Tests/Util/XCTestCase+Await.mm', + 'Firestore/Example/Tests/en.lproj/InfoPlist.strings', + ] + end + + s.sync() + sort_project(project) + if project.dirty? + project.save() + end +end + + +# The definition of a test target including the target name, its source_files +# and exclude_files. A file is considered part of a target if it matches a +# pattern in source_files but does not match a pattern in exclude_files. +class TargetDef + def initialize(name) + @name = name + @source_files = [] + @exclude_files = [] + end + + attr_accessor :name, :source_files, :exclude_files + + # Returns true if the given relative_path matches this target's source_files + # but not its exclude_files. + # + # Args: + # - relative_path: a Pathname instance with a path relative to the project + # root. + def matches?(relative_path) + return matches_patterns(relative_path, @source_files) && + !matches_patterns(relative_path, @exclude_files) + end + + private + # Evaluates the relative_path against the given list of fnmatch patterns. + def matches_patterns(relative_path, patterns) + patterns.each do |pattern| + if relative_path.fnmatch?(pattern) + return true + end + end + return false + end +end + + +class Syncer + def initialize(project, root_dir) + @project = project + @root_dir = Pathname.new(root_dir) + + @finder = DirectoryLister.new(@root_dir) + + @seen_groups = {} + + @test_groups = [] + @targets = [] + end + + # Considers the given fnmatch glob patterns to be ignored by the syncer. + # Patterns are matched both against the basename and project-relative + # qualified pathname. + def ignore_files=(patterns) + @finder.add_patterns(patterns) + end + + # Names the groups within the project that serve as roots for tests within + # the project. + def test_groups=(groups) + @test_groups = [] + groups.each do |group| + project_group = @project[group] + if project_group.nil? + raise "Project does not contain group #{group}" + end + @test_groups.push(@project[group]) + end + end + + # Starts a new target block. Creates a new TargetDef and yields it. + def target(name, &block) + t = TargetDef.new(name) + @targets.push(t) + + block.call(t) + end + + # Synchronizes the filesystem with the project. + # + # Generally there are three separate ways a file is referenced within a project: + # + # 1. The file must be in the global list of files, assigning it a UUID. + # 2. The file must be added to folder groups, describing where it is in the + # folder view of the Project Navigator. + # 3. The file must be added to a target descrbing how it's built. + # + # The Xcodeproj library handles (1) for us automatically if we do (2). + # + # Synchronization essentially proceeds in two steps: + # + # 1. Sync the filesystem structure with the folder group structure. This has + # the effect of bringing (1) and (2) into sync. + # 2. Sync the global list of files with the targets. + def sync() + group_differ = GroupDiffer.new(@finder) + group_diffs = group_differ.diff(@test_groups) + sync_groups(group_diffs) + + @targets.each do |target_def| + sync_target(target_def) + end + end + + private + def sync_groups(diff_entries) + diff_entries.each do |entry| + if !entry.in_source && entry.in_target + remove_from_project(entry.ref) + end + + if entry.in_source && !entry.in_target + add_to_project(entry.path) + end + end + end + + # Removes the given file reference from the project after the file is found + # missing but references to it still exist in the project. + def remove_from_project(file_ref) + group = file_ref.parents[-1] + + mark_change_in_group(relative_path(group)) + puts " #{basename(file_ref)} - removed" + + # If the file is gone, any build phase that refers to must also remove the + # file. Without this, the project will have build file references that + # contain no actual file. + @project.native_targets.each do |target| + target.build_phases.each do |phase| + if phase.include?(file_ref) + phase.remove_file_reference(file_ref) + end + end + end + + file_ref.remove_from_project + end + + # Adds the given file to the project, in a path starting from the test root + # that fully prefixes the file. + def add_to_project(path) + root_group = find_test_group_containing(path) + + # Find or create the group to contain the path. + dir_rel_path = path.relative_path_from(root_group.real_path).dirname + group = root_group.find_subpath(dir_rel_path.to_s, true) + + mark_change_in_group(relative_path(group)) + + file_ref = group.new_file(path.to_s) + + puts " #{basename(file_ref)} - added" + return file_ref + end + + # Finds a test group whose path prefixes the given entry. Starting from the + # project root may not work since not all test directories exist within the + # example app. + def find_test_group_containing(path) + @test_groups.each do |group| + rel = path.relative_path_from(group.real_path) + next if rel.to_s.start_with?('..') + + return group + end + + raise "Could not find an existing test group that's a parent of #{entry.path}" + end + + def mark_change_in_group(group) + path = group.to_s + if !@seen_groups.has_key?(path) + puts "#{path} ..." + @seen_groups[path] = true + end + end + + SOURCES = %w{.c .cc .m .mm} + + def sync_target(target_def) + target = @project.native_targets.find { |t| t.name == target_def.name } + if !target + raise "Missing target #{target_def.name}" + end + + files = find_files_for_target(target_def) + sources, resources = classify_files(files) + + sync_build_phase(target, target.source_build_phase, sources) + end + + def classify_files(files) + sources = {} + resources = {} + + files.each do |file| + path = file.real_path + ext = path.extname + if SOURCES.include?(ext) + sources[path] = file + end + end + + return sources, resources + end + + def sync_build_phase(target, phase, sources) + # buffer changes to the phase to avoid modifying the array we're iterating + # over. + to_remove = [] + phase.files.each do |build_file| + source_path = build_file.file_ref.real_path + if sources.has_key?(source_path) + # matches spec and existing target no action taken + sources.delete(source_path) + + else + # in the phase but now missing in the groups + to_remove.push(build_file) + end + end + + to_remove.each do |build_file| + mark_change_in_group(target.name) + + source_path = build_file.file_ref.real_path + puts " #{relative_path(source_path)} - removed" + phase.remove_build_file(build_file) + end + + sources.each do |path, file_ref| + mark_change_in_group(target.name) + + phase.add_file_reference(file_ref) + puts " #{relative_path(file_ref)} - added" + end + end + + def find_files_for_target(target_def) + result = [] + + @project.files.each do |file_ref| + next if file_ref.source_tree != '' + + rel = relative_path(file_ref) + if target_def.matches?(rel) + result.push(file_ref) + end + end + return result + end + + def normalize_to_pathname(file_ref) + if !file_ref.is_a? Pathname + if file_ref.is_a? String + file_ref = Pathname.new(file_ref) + else + file_ref = file_ref.real_path + end + end + return file_ref + end + + def basename(file_ref) + return normalize_to_pathname(file_ref).basename + end + + def relative_path(file_ref) + file_ref = normalize_to_pathname(file_ref) + return file_ref.relative_path_from(@root_dir) + end +end + + +def sort_project(project) + project.groups.each do |group| + sort_group(group) + end + + project.targets.each do |target| + target.build_phases.each do |phase| + phase.files.sort! { |a, b| + a.file_ref.real_path.basename <=> b.file_ref.real_path.basename + } + end + end +end + + +def sort_group(group) + group.groups.each do |child| + sort_group(child) + end + + group.children.sort! do |a, b| + # Sort groups first + if a.isa == 'PBXGroup' && b.isa != 'PBXGroup' + -1 + elsif a.isa != 'PBXGroup' && b.isa == 'PBXGroup' + 1 + elsif a.display_name && b.display_name + File.basename(a.display_name) <=> File.basename(b.display_name) + else + 0 + end + end +end + + +# Tracks how a file is referenced: in the project file, on the filesystem, +# neither, or both. +class DiffEntry + def initialize(path) + @path = path + @in_source = false + @in_target = false + @ref = nil + end + + attr_reader :path + attr_accessor :in_source, :in_target, :ref +end + + +# Diffs folder groups against the filesystem directories referenced by those +# folder groups. +# +# This performs the diff starting from the directories referenced by the test +# groups in the project, finding files contained within them. When comparing +# the files it finds against the project this acts on absolute paths to avoid +# problems with arbitary additional groupings in project structure that are +# standard, e.g. "Supporting Files" or "en.lproj" which either act as aliases +# for the parent or are folders that are omitted from the project view. +# Processing the diff this way allows these warts to be tolerated, even if they +# won't necessarily be recreated if an artifact is added to the filesystem. +class GroupDiffer + def initialize(dir_lister) + @dir_lister = dir_lister + + @entries = {} + @dirs = {} + end + + # Finds all tests on the filesystem contained within the paths of the given + # test groups and computes a list of DiffEntries describing the state of the + # files. + # + # Args: + # - groups: A list of PBXGroup objects representing folder groups within the + # project that contain tests. + # + # Returns: + # A list of DiffEntry objects, one for each test found. If the test exists on + # the filesystem, :in_source will be true. If the test exists in the project + # :in_target will be true and :ref will be set to the PBXFileReference naming + # the file. + def diff(groups) groups.each do |group| diff_project_files(group) end + + return @entries.values.sort { |a, b| a.path.basename <=> b.path.basename } + end + + private + # Recursively traverses all the folder groups in the Xcode project and finds + # files both on the filesystem and the group file listing. + def diff_project_files(group) + find_fs_files(group.real_path) + + group.groups.each do |child| + diff_project_files(child) + end + + group.files.each do |file_ref| + path = file_ref.real_path + entry = track_file(path) + entry.in_target = true + entry.ref = file_ref + + if path.file? + entry.in_source = true + end + end + end + + def find_fs_files(parent_path) + # Avoid re-traversing the filesystem + if @dirs.has_key?(parent_path) + return + end + @dirs[parent_path] = true + + @dir_lister.entries(parent_path).each do |path| + if path.directory? + find_fs_files(path) + next + end + + entry = track_file(path) + entry.in_source = true + end + end + + def track_file(path) + if @entries.has_key?(path) + return @entries[path] + end + + entry = DiffEntry.new(path) + @entries[path] = entry + return entry + end +end + + +# Finds files on the filesystem while ignoring files that have been declared to +# be ignored. +class DirectoryLister + def initialize(root_dir) + @root_dir = root_dir + @ignore_basenames = ['.', '..'] + @ignore_pathnames = [] + end + + def add_patterns(patterns) + patterns.each do |pattern| + if File.basename(pattern) != pattern + @ignore_pathnames.push(File.join(@root_dir, pattern)) + else + @ignore_basenames.push(pattern) + end + end + end + + # Finds filesystem entries that are immediate children of the given Pathname, + # ignoring files that match the the global ignore_files patterns. + def entries(path) + result = [] + path.entries.each do |entry| + next if ignore_basename?(entry) + + file = path.join(entry) + next if ignore_pathname?(file) + + result.push(file) + end + return result + end + + private + def ignore_basename?(basename) + @ignore_basenames.each do |ignore| + if basename.fnmatch(ignore) + return true + end + end + return false + end + + def ignore_pathname?(file) + @ignore_pathnames.each do |ignore| + if file.fnmatch(ignore) + return true + end + end + return false + end +end + + +if __FILE__ == $0 + main() +end -- cgit v1.2.3