Keep "special case" modifications when regenerating project files

The pro2cmake.py file is now smarter, and can reapply "special case"
modifications after regenerating a CMakeLists.txt file.

This substantially lowers the maintenance burden when regenerating
many files at once.

See the special_case_helper.py file for details on how it works.

Make sure to commit the generated .prev_CMakeLists.txt file alongside
your CMakeLists.txt changes.

To disable the preservation behavior, you can pass -s or
--skip-special-case-preservation to the script.

To keep around temporary files that are created during this process,
you can pass -k or --keep-temporary-files.

To get more debug output, pass --debug-special-case-preservation.

Fixes: QTBUG-75619
Change-Id: I6d8ba52ac5feb5020f31d47841203104c2a061d8
Reviewed-by: Tobias Hunger <tobias.hunger@qt.io>

util/cmake/pro2cmake.py

@@ -45,6 +45,9 @@ import pyparsing as pp
 from helper import map_qt_library, map_3rd_party_library, is_known_3rd_party_library, \
     featureName, map_platform
 
+from shutil import copyfile
+from special_case_helper import SpecialCaseHandler
+
 
 def _parse_commandline():
     parser = ArgumentParser(description='Generate CMakeLists.txt files from .'
@@ -67,10 +70,21 @@ def _parse_commandline():
                         dest='debug_full_pro_structure', action='store_true',
                         help='Dump the full structure of the qmake .pro-file '
                         '(with includes).')
+    parser.add_argument('--debug-special-case-preservation',
+                        dest='debug_special_case_preservation', action='store_true',
+                        help='Show all git commands and file copies.')
 
     parser.add_argument('--example', action='store_true',
                         dest="is_example",
                         help='Treat the input .pro file as an example.')
+    parser.add_argument('-s', '--skip-special-case-preservation',
+                        dest='skip_special_case_preservation', action='store_true',
+                        help='Skips behavior to reapply '
+                             'special case modifications (requires git in PATH)')
+    parser.add_argument('-k', '--keep-temporary-files',
+                        dest='keep_temporary_files', action='store_true',
+                        help='Don\'t automatically remove CMakeLists.gen.txt and other '
+                             'intermediate files.')
 
     parser.add_argument('files', metavar='<.pro/.pri file>', type=str,
                         nargs='+', help='The .pro/.pri file to process')
@@ -448,7 +462,12 @@ class Scope(object):
         return self._file or ''
 
     @property
-    def cMakeListsFile(self) -> str:
+    def generated_cmake_lists_path(self) -> str:
+        assert self.basedir
+        return os.path.join(self.basedir, 'CMakeLists.gen.txt')
+
+    @property
+    def original_cmake_lists_path(self) -> str:
         assert self.basedir
         return os.path.join(self.basedir, 'CMakeLists.txt')
 
@@ -1596,8 +1615,9 @@ def cmakeify_scope(scope: Scope, cm_fh: typing.IO[str], *,
               .format(scope.file, template))
 
 
-def generate_cmakelists(scope: Scope, *, is_example: bool=False) -> None:
+def generate_new_cmakelists(scope: Scope, *, is_example: bool=False) -> None:
-    with open(scope.cMakeListsFile, 'w') as cm_fh:
+    print('Generating CMakeLists.gen.txt')
+    with open(scope.generated_cmake_lists_path, 'w') as cm_fh:
         assert scope.file
         cm_fh.write('# Generated from {}.\n\n'
                     .format(os.path.basename(scope.file)))
@@ -1626,6 +1646,14 @@ def do_include(scope: Scope, *, debug: bool = False) -> None:
         scope.merge(include_scope)
 
 
+def copy_generated_file_to_final_location(scope: Scope, keep_temporary_files=False) -> None:
+    print('Copying {} to {}'.format(scope.generated_cmake_lists_path,
+                                    scope.original_cmake_lists_path))
+    copyfile(scope.generated_cmake_lists_path, scope.original_cmake_lists_path)
+    if not keep_temporary_files:
+        os.remove(scope.generated_cmake_lists_path)
+
+
 def main() -> None:
     args = _parse_commandline()
 
@@ -1665,7 +1693,22 @@ def main() -> None:
             file_scope.dump()
             print('\n#### End of full .pro/.pri file structure.\n')
 
-        generate_cmakelists(file_scope, is_example=args.is_example)
+        generate_new_cmakelists(file_scope, is_example=args.is_example)
+
+        copy_generated_file = True
+        if not args.skip_special_case_preservation:
+            debug_special_case = args.debug_special_case_preservation or args.debug
+            handler = SpecialCaseHandler(file_scope.original_cmake_lists_path,
+                                         file_scope.generated_cmake_lists_path,
+                                         file_scope.basedir,
+                                         keep_temporary_files=args.keep_temporary_files,
+                                         debug=debug_special_case)
+
+            copy_generated_file = handler.handle_special_cases()
+
+        if copy_generated_file:
+            copy_generated_file_to_final_location(file_scope,
+                                                  keep_temporary_files=args.keep_temporary_files)
         os.chdir(backup_current_dir)
 
 

util/cmake/special_case_helper.py

@@ -0,0 +1,347 @@
+#!/usr/bin/env python3
+#############################################################################
+##
+## Copyright (C) 2019 The Qt Company Ltd.
+## Contact: https://www.qt.io/licensing/
+##
+## This file is part of the plugins of the Qt Toolkit.
+##
+## $QT_BEGIN_LICENSE:GPL-EXCEPT$
+## Commercial License Usage
+## Licensees holding valid commercial Qt licenses may use this file in
+## accordance with the commercial license agreement provided with the
+## Software or, alternatively, in accordance with the terms contained in
+## a written agreement between you and The Qt Company. For licensing terms
+## and conditions see https://www.qt.io/terms-conditions. For further
+## information use the contact form at https://www.qt.io/contact-us.
+##
+## GNU General Public License Usage
+## Alternatively, this file may be used under the terms of the GNU
+## General Public License version 3 as published by the Free Software
+## Foundation with exceptions as appearing in the file LICENSE.GPL3-EXCEPT
+## included in the packaging of this file. Please review the following
+## information to ensure the GNU General Public License requirements will
+## be met: https://www.gnu.org/licenses/gpl-3.0.html.
+##
+## $QT_END_LICENSE$
+##
+#############################################################################
+
+"""
+This is a helper script that takes care of reapplying special case
+modifications when regenerating a CMakeLists.txt file using
+pro2cmake.py.
+
+It has two modes of operation:
+1) Dumb "special case" block removal and re-application.
+2) Smart "special case" diff application, using a previously generated
+   "clean" CMakeLists.txt as a source. "clean" in this case means a
+   generated file which has no "special case" modifications.
+
+Both modes use a temporary git repository to compute and reapply
+"special case" diffs.
+
+For the first mode to work, the developer has to mark changes
+with "# special case" markers on every line they want to keep. Or
+enclose blocks of code they want to keep between "# special case begin"
+and "# special case end" markers.
+
+For example:
+
+SOURCES
+  foo.cpp
+  bar.cpp # special case
+
+SOURCES
+  foo1.cpp
+  foo2.cpp
+ # special case begin
+  foo3.cpp
+  foo4.cpp
+ # special case end
+
+The second mode, as mentioned, requires a previous "clean"
+CMakeLists.txt file.
+
+The script can then compute the exact diff between
+a "clean" and "modified" (with special cases) file, and reapply that
+diff to a newly generated "CMakeLists.txt" file.
+
+This implies that we always have to keep a "clean" file alongside the
+"modified" project file for each project (corelib, gui, etc.) So we
+have to commit both files to the repository.
+
+If there is no such "clean" file, we can use the first operation mode
+to generate one. After that, we only have to use the second operation
+mode for the project file in question.
+
+When the script is used, the developer only has to take care of fixing
+the newly generated "modified" file. The "clean" file is automatically
+handled and git add'ed by the script, and will be committed together
+with the "modified" file.
+
+
+"""
+
+import re
+import os
+import subprocess
+import filecmp
+
+from shutil import copyfile
+from shutil import rmtree
+
+
+def remove_special_cases(original: str) -> str:
+    # Remove content between the following markers
+    # '# special case begin' and '# special case end'.
+    # This also remove the markers.
+    replaced = re.sub(r'\n[^#\n]*?#[^\n]*?special case begin.*?#[^\n]*special case end[^\n]*?\n',
+                      '\n',
+                      original,
+                      0,
+                      re.DOTALL)
+
+    # Remove individual lines that have the "# special case" marker.
+    replaced = re.sub(r'\n.*#.*special case[^\n]*\n', '\n', replaced)
+    return replaced
+
+
+def read_content_from_file(file_path: str) -> str:
+    with open(file_path, 'r') as file_fd:
+        content = file_fd.read()
+        return content
+
+
+def write_content_to_file(file_path: str, content: str) -> None:
+    with open(file_path, 'w') as file_fd:
+        file_fd.write(content)
+
+
+def resolve_simple_git_conflicts(file_path: str, debug=False) -> None:
+    content = read_content_from_file(file_path)
+    # If the conflict represents the addition of a new content hunk,
+    # keep the content and remove the conflict markers.
+    if debug:
+        print('Resolving simple conflicts automatically.')
+    replaced = re.sub(r'\n<<<<<<< HEAD\n=======(.+?)>>>>>>> master\n', r'\1', content, 0, re.DOTALL)
+    write_content_to_file(file_path, replaced)
+
+
+def copyfile_log(src: str, dst: str, debug=False):
+    if debug:
+        print('Copying {} to {}.'.format(src, dst))
+    copyfile(src, dst)
+
+
+def check_if_git_in_path() -> bool:
+    for path in os.environ['PATH'].split(os.pathsep):
+        git_path = os.path.join(path, 'git')
+        if os.path.isfile(git_path) and os.access(git_path, os.X_OK):
+            return True
+    return False
+
+
+def run_process_quiet(args_string: str, debug=False) -> None:
+    if debug:
+        print('Running command: "{}\"'.format(args_string))
+    args_list = args_string.split()
+    subprocess.run(args_list, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+
+def does_file_have_conflict_markers(file_path: str, debug=False) -> bool:
+    if debug:
+        print('Checking if {} has no leftover conflict markers.'.format(file_path))
+    content_actual = read_content_from_file(file_path)
+    if '<<<<<<< HEAD' in content_actual:
+        print('Conflict markers found in {}. '
+              'Please remove or solve them first.'.format(file_path))
+        return True
+    return False
+
+
+def create_file_with_no_special_cases(original_file_path: str, no_special_cases_file_path: str, debug=False):
+    """
+    Reads content of original CMakeLists.txt, removes all content
+    between "# special case" markers or lines, saves the result into a
+    new file.
+    """
+    content_actual = read_content_from_file(original_file_path)
+    if debug:
+        print('Removing special case blocks from {}.'.format(original_file_path))
+    content_no_special_cases = remove_special_cases(content_actual)
+
+    if debug:
+        print('Saving original contents of {} '
+              'with removed special case blocks to {}'.format(original_file_path,
+                                                          no_special_cases_file_path))
+    write_content_to_file(no_special_cases_file_path, content_no_special_cases)
+
+
+class SpecialCaseHandler(object):
+
+    def __init__(self,
+                 original_file_path: str,
+                 generated_file_path: str,
+                 base_dir: str,
+                 keep_temporary_files=False,
+                 debug=False) -> None:
+        self.base_dir = base_dir
+        self.original_file_path = original_file_path
+        self.generated_file_path = generated_file_path
+        self.keep_temporary_files = keep_temporary_files
+        self.use_heuristic = False
+        self.debug = debug
+
+    @property
+    def prev_file_path(self) -> str:
+        return os.path.join(self.base_dir, '.prev_CMakeLists.txt')
+
+    @property
+    def post_merge_file_path(self) -> str:
+        return os.path.join(self.base_dir, 'CMakeLists-post-merge.txt')
+
+    @property
+    def no_special_file_path(self) -> str:
+        return os.path.join(self.base_dir, 'CMakeLists.no-special.txt')
+
+    def apply_git_merge_magic(self, no_special_cases_file_path: str) -> None:
+        # Create new folder for temporary repo, and ch dir into it.
+        repo = os.path.join(self.base_dir, 'tmp_repo')
+        repo_absolute_path = os.path.abspath(repo)
+        txt = 'CMakeLists.txt'
+
+        try:
+            os.mkdir(repo)
+            current_dir = os.getcwd()
+            os.chdir(repo)
+        except Exception as e:
+            print('Failed to create temporary directory for temporary git repo. Exception: {}'
+                  .format(e))
+            raise e
+
+        generated_file_path = os.path.join("..", self.generated_file_path)
+        original_file_path = os.path.join("..", self.original_file_path)
+        no_special_cases_file_path = os.path.join("..", no_special_cases_file_path)
+        post_merge_file_path = os.path.join("..", self.post_merge_file_path)
+
+        try:
+            # Create new repo with the "clean" CMakeLists.txt file.
+            run_process_quiet('git init .', debug=self.debug)
+            copyfile_log(no_special_cases_file_path, txt, debug=self.debug)
+            run_process_quiet('git add {}'.format(txt), debug=self.debug)
+            run_process_quiet('git commit -m no_special', debug=self.debug)
+
+            # Copy the original "modified" file (with the special cases)
+            # and make a new commit.
+            copyfile_log(original_file_path, txt, debug=self.debug)
+            run_process_quiet('git add {}'.format(txt), debug=self.debug)
+            run_process_quiet('git commit -m original', debug=self.debug)
+
+            # Checkout the commit with "clean" file again, and create a
+            # new branch.
+            run_process_quiet('git checkout HEAD~', debug=self.debug)
+            run_process_quiet('git checkout -b newly_generated', debug=self.debug)
+
+            # Copy the new "modified" file and make a commit.
+            copyfile_log(generated_file_path, txt, debug=self.debug)
+            run_process_quiet('git add {}'.format(txt), debug=self.debug)
+            run_process_quiet('git commit -m newly_generated', debug=self.debug)
+
+            # Merge the "old" branch with modifications into the "new"
+            # branch with the newly generated file.
+            run_process_quiet('git merge master', debug=self.debug)
+
+            # Resolve some simple conflicts (just remove the markers)
+            # for cases that don't need intervention.
+            resolve_simple_git_conflicts(txt, debug=self.debug)
+
+            # Copy the resulting file from the merge.
+            copyfile_log(txt, post_merge_file_path)
+        except Exception as e:
+            print('Git merge conflict resolution process failed. Exception: {}'.format(e))
+            raise e
+        finally:
+            # Remove the temporary repo.
+            try:
+                if not self.keep_temporary_files:
+                    rmtree(repo_absolute_path)
+            except Exception as e:
+                print(e)
+
+            os.chdir(current_dir)
+
+    def save_next_clean_file(self):
+        files_are_equivalent = filecmp.cmp(self.generated_file_path, self.post_merge_file_path)
+
+        if not files_are_equivalent:
+            # Before overriding the generated file with the post
+            # merge result, save the new "clean" file for future
+            # regenerations.
+            copyfile_log(self.generated_file_path, self.prev_file_path, debug=self.debug)
+            run_process_quiet("git add {}".format(self.prev_file_path), debug=self.debug)
+
+    def handle_special_cases_helper(self) -> bool:
+        """
+        Uses git to reapply special case modifications to the "new"
+        generated CMakeLists.gen.txt file.
+
+        If use_heuristic is True, a new file is created from the
+        original file, with special cases removed.
+
+        If use_heuristic is False, an existing "clean" file with no
+        special cases is used from a previous conversion. The "clean"
+        file is expected to be in the same folder as the original one.
+        """
+        try:
+            if does_file_have_conflict_markers(self.original_file_path):
+                return False
+
+            if self.use_heuristic:
+                create_file_with_no_special_cases(self.original_file_path,
+                                                  self.no_special_file_path)
+                no_special_cases_file_path = self.no_special_file_path
+            else:
+                no_special_cases_file_path = self.prev_file_path
+
+            if self.debug:
+                print('Using git to reapply special case modifications to newly generated {} '
+                      'file'.format(self.generated_file_path))
+
+            self.apply_git_merge_magic(no_special_cases_file_path)
+            self.save_next_clean_file()
+
+            copyfile_log(self.post_merge_file_path, self.generated_file_path)
+            if not self.keep_temporary_files:
+                os.remove(self.post_merge_file_path)
+
+            print('Special case reapplication using git is complete. '
+                  'Make sure to fix remaining conflict markers.')
+
+        except Exception as e:
+            print('Error occurred while trying to reapply special case modifications: {}'.format(e))
+            return False
+        finally:
+            if not self.keep_temporary_files and self.use_heuristic:
+                os.remove(self.no_special_file_path)
+
+        return True
+
+    def handle_special_cases(self) -> bool:
+        original_file_exists = os.path.isfile(self.original_file_path)
+        prev_file_exists = os.path.isfile(self.prev_file_path)
+        self.use_heuristic = not prev_file_exists
+
+        git_available = check_if_git_in_path()
+        keep_special_cases = original_file_exists and git_available
+
+        if not git_available:
+            print('You need to have git in PATH in order to reapply the special '
+                  'case modifications.')
+
+        copy_generated_file = True
+
+        if keep_special_cases:
+            copy_generated_file = self.handle_special_cases_helper()
+
+        return copy_generated_file