Merge commit '8338b786b05a9e2160a3020b6e59154f5dcd4813'

manage_externals/.github/workflows/bumpversion.yml

@@ -0,0 +1,19 @@
+name: Bump version
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Bump version and push tag
+        id: tag_version
+        uses: mathieudutour/[email protected]
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          create_annotated_tag: true
+          default_bump: patch
+          dry_run: false
+          tag_prefix: manic-

manage_externals/.github/workflows/tests.yml

@@ -0,0 +1,30 @@
+# This is a workflow to compile the cmeps source without cime
+name: Test Manic
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  test-manic:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Test Manic
+        run: |
+          pushd test
+          git config --global user.email "[email protected]"
+          git config --global user.name "GITHUB tester"
+          git config --global protocol.file.allow always
+          make utest
+          make stest
+          popd
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

manage_externals/README.md

@@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     externals description file pointed 'useful_library/sub-xternals.cfg',
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
+    Note that by default, `checkout_externals` will clone an external's
+    submodules. As a special case, the entry, `externals = None`, will
+    prevent this behavior. For more control over which externals are
+    checked out, create an externals file (and see the `from_submodule`
+    configuration entry below).
 
   * from_submodule (True / False) : used to pull the repo_url, local_path,
     and hash properties for this external from the .gitmodules file in

manage_externals/manic/checkout.py

@@ -227,6 +227,12 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+    Note that by default, checkout_externals will clone an external's
+    submodules. As a special case, the entry, "externals = None", will
+    prevent this behavior. For more control over which externals are
+    checked out, create an externals file (and see the from_submodule
+    configuration entry below).
+
   * from_submodule (True / False) : used to pull the repo_url, local_path,
     and hash properties for this external from the .gitmodules file in
     this repository. Note that the section name (the entry in square
@@ -332,7 +338,34 @@ def commandline_arguments(args=None):
         options = parser.parse_args()
     return options
 
-
+def _dirty_local_repo_msg(program_name, config_file):
+    return """The external repositories labeled with 'M' above are not in a clean state.
+The following are four options for how to proceed:
+(1) Go into each external that is not in a clean state and issue either a 'git status' or
+    an 'svn status' command (depending on whether the external is managed by git or
+    svn). Either revert or commit your changes so that all externals are in a clean
+    state. (To revert changes in git, follow the instructions given when you run 'git
+    status'.) (Note, though, that it is okay to have untracked files in your working
+    directory.) Then rerun {program_name}.
+(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
+    update out-of-sync externals (labeled with 's' above) as described in the
+    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
+    commands to checkout the appropriate tags for each external, as given in
+    {config_file}.)
+(3) You can also use {program_name} to manage most, but not all externals: You can specify
+    one or more externals to ignore using the '-x' or '--exclude' argument to
+    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
+    update the other, non-excluded externals.
+(4) As a last resort, if you are confident that there is no work that needs to be saved
+    from a given external, you can remove that external (via "rm -rf [directory]") and
+    then rerun the {program_name} tool. This option is mainly useful as a workaround for
+    issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157).
+The external repositories labeled with '?' above are not under version
+control using the expected protocol. If you are sure you want to switch
+protocols, and you don't have any work you need to save from this
+directory, then run "rm -rf [directory]" before rerunning the
+{program_name} tool.
+""".format(program_name=program_name, config_file=config_file)
 # ---------------------------------------------------------------------
 #
 # main
@@ -363,19 +396,25 @@ def main(args):
         load_all = True
 
     root_dir = os.path.abspath(os.getcwd())
-    external_data = read_externals_description_file(root_dir, args.externals)
-    external = create_externals_description(
-        external_data, components=args.components, exclude=args.exclude)
+    model_data = read_externals_description_file(root_dir, args.externals)
+    ext_description = create_externals_description(
+        model_data, components=args.components, exclude=args.exclude)
 
     for comp in args.components:
-        if comp not in external.keys():
+        if comp not in ext_description.keys():
+            # Note we can't print out the list of found externals because
+            # they were filtered in create_externals_description above.
             fatal_error(
                 "No component {} found in {}".format(
                     comp, args.externals))
 
-    source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry)
-    printlog('Checking status of externals: ', end='')
-    tree_status = source_tree.status()
+    source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry)
+    if args.components:
+        components_str = 'specified components'
+    else:
+        components_str = 'required & optional components'
+    printlog('Checking local status of ' + components_str + ': ', end='')
+    tree_status = source_tree.status(print_progress=True)
     printlog('')
 
     if args.status:
@@ -390,43 +429,8 @@ def main(args):
             for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
-            msg = """The external repositories labeled with 'M' above are not in a clean state.
-
-The following are four options for how to proceed:
-
-(1) Go into each external that is not in a clean state and issue either a 'git status' or
-    an 'svn status' command (depending on whether the external is managed by git or
-    svn). Either revert or commit your changes so that all externals are in a clean
-    state. (To revert changes in git, follow the instructions given when you run 'git
-    status'.) (Note, though, that it is okay to have untracked files in your working
-    directory.) Then rerun {program_name}.
-
-(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
-    update out-of-sync externals (labeled with 's' above) as described in the
-    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
-    commands to checkout the appropriate tags for each external, as given in
-    {config_file}.)
-
-(3) You can also use {program_name} to manage most, but not all externals: You can specify
-    one or more externals to ignore using the '-x' or '--exclude' argument to
-    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
-    update the other, non-excluded externals.
-
-(4) As a last resort, if you are confident that there is no work that needs to be saved
-    from a given external, you can remove that external (via "rm -rf [directory]") and
-    then rerun the {program_name} tool. This option is mainly useful as a workaround for
-    issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157).
-
-
-The external repositories labeled with '?' above are not under version
-control using the expected protocol. If you are sure you want to switch
-protocols, and you don't have any work you need to save from this
-directory, then run "rm -rf [directory]" before rerunning the
-{program_name} tool.
-""".format(program_name=program_name, config_file=args.externals)
-
             printlog('-' * 70)
-            printlog(msg)
+            printlog(_dirty_local_repo_msg(program_name, args.externals))
             printlog('-' * 70)
         else:
             if not args.components:

manage_externals/manic/externals_description.py

@@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name):
     root_dir = os.path.abspath(root_dir)
     msg = 'In directory : {0}'.format(root_dir)
     logging.info(msg)
-    printlog('Processing externals description file : {0}'.format(file_name))
+    printlog('Processing externals description file : {0} ({1})'.format(file_name,
+                                                                        root_dir))
 
     file_path = os.path.join(root_dir, file_name)
     if not os.path.exists(file_name):
@@ -281,6 +282,10 @@ def read_gitmodules_file(root_dir, file_name):
 def create_externals_description(
         model_data, model_format='cfg', components=None, exclude=None, parent_repo=None):
     """Create the a externals description object from the provided data
+        
+    components: list of component names to include, None to include all. If a
+                name isn't found, it is silently omitted from the return value.
+    exclude: list of component names to skip.
     """
     externals_description = None
     if model_format == 'dict':
@@ -357,8 +362,9 @@ class ExternalsDescription(dict):
     input value.
 
     """
-    # keywords defining the interface into the externals description data
-    EXTERNALS = 'externals'
+    # keywords defining the interface into the externals description data; these
+    # are brought together by the schema below.
+    EXTERNALS = 'externals'  # path to externals file.
     BRANCH = 'branch'
     SUBMODULE = 'from_submodule'
     HASH = 'hash'
@@ -384,6 +390,8 @@ class ExternalsDescription(dict):
     _V1_BRANCH = 'BRANCH'
     _V1_REQ_SOURCE = 'REQ_SOURCE'
 
+    # Dictionary keys are component names. The corresponding values are laid out
+    # according to this schema.
     _source_schema = {REQUIRED: True,
                       PATH: 'string',
                       EXTERNALS: 'string',
@@ -760,6 +768,8 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None):
         """Convert the config data into a standardized dict that can be used to
         construct the source objects
 
+        components: list of component names to include, None to include all.
+        exclude: list of component names to skip.
         """
         ExternalsDescription.__init__(self, parent_repo=parent_repo)
         self._schema_major = 1
@@ -783,6 +793,9 @@ def _remove_metadata(model_data):
 
     def _parse_cfg(self, cfg_data, components=None, exclude=None):
         """Parse a config_parser object into a externals description.
+
+        components: list of component names to include, None to include all.
+        exclude: list of component names to skip.
         """
         def list_to_dict(input_list, convert_to_lower_case=True):
             """Convert a list of key-value pairs into a dictionary.

manage_externals/manic/externals_status.py

@@ -29,16 +29,16 @@ class ExternalStatus(object):
     transactions (e.g. add, remove, rename, untracked files).
 
     """
-    DEFAULT = '-'
+    # sync_state and clean_state can be one of the following:
+    DEFAULT = '-'  # aka not set yet.
     UNKNOWN = '?'
     EMPTY = 'e'
-    MODEL_MODIFIED = 's'  # a.k.a. out-of-sync
-    DIRTY = 'M'
-
-    STATUS_OK = ' '
+    MODEL_MODIFIED = 's'  # repo version != externals (sync_state only)
+    DIRTY = 'M'       # repo is dirty (clean_state only)
+    STATUS_OK = ' '   # repo is clean/matches externals.
     STATUS_ERROR = '!'
 
-    # source types
+    # source_type can be one of the following:
     OPTIONAL = 'o'
     STANDALONE = 's'
     MANAGED = ' '
@@ -55,19 +55,21 @@ def __init__(self):
     def log_status_message(self, verbosity):
         """Write status message to the screen and log file
         """
-        self._default_status_message()
+        printlog(self._default_status_message())
         if verbosity >= VERBOSITY_VERBOSE:
-            self._verbose_status_message()
+            printlog(self._verbose_status_message())
         if verbosity >= VERBOSITY_DUMP:
-            self._dump_status_message()
+            printlog(self._dump_status_message())
+
+    def __repr__(self):
+        return self._default_status_message()
 
     def _default_status_message(self):
         """Return the default terse status message string
         """
-        msg = '{sync}{clean}{src_type} {path}'.format(
+        return '{sync}{clean}{src_type} {path}'.format(
             sync=self.sync_state, clean=self.clean_state,
             src_type=self.source_type, path=self.path)
-        printlog(msg)
 
     def _verbose_status_message(self):
         """Return the verbose status message string
@@ -82,14 +84,12 @@ def _verbose_status_message(self):
         if self.sync_state != self.STATUS_OK:
             sync_str = '{current} --> {expected}'.format(
                 current=self.current_version, expected=self.expected_version)
-        msg = '        {clean}, {sync}'.format(clean=clean_str, sync=sync_str)
-        printlog(msg)
+        return '        {clean}, {sync}'.format(clean=clean_str, sync=sync_str)
 
     def _dump_status_message(self):
         """Return the dump status message string
         """
-        msg = indent_string(self.status_output, 12)
-        printlog(msg)
+        return indent_string(self.status_output, 12)
 
     def safe_to_update(self):
         """Report if it is safe to update a repository. Safe is defined as:

manage_externals/manic/repository_factory.py

@@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False):
     """Determine what type of repository we have, i.e. git or svn, and
     create the appropriate object.
 
+    Can return None (e.g. if protocol is 'externals_only').
     """
     protocol = repo_info[ExternalsDescription.PROTOCOL].lower()
     if protocol == 'git':

manage_externals/manic/repository_svn.py

@@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False):
         """
         Repository.__init__(self, component_name, repo)
         self._ignore_ancestry = ignore_ancestry
+        if self._url.endswith('/'):
+            # there is already a '/' separator in the URL; no need to add another
+            url_sep = ''
+        else:
+            url_sep = '/'
         if self._branch:
-            self._url = os.path.join(self._url, self._branch)
+            self._url = self._url + url_sep + self._branch
         elif self._tag:
-            self._url = os.path.join(self._url, self._tag)
+            self._url = self._url + url_sep + self._tag
         else:
             msg = "DEV_ERROR in svn repository. Shouldn't be here!"
             fatal_error(msg)