Merge branch 'skip-imports' into 'dev'

Make CLI bootstrapping faster

See merge request epi2melabs/workflows/wf-template!206

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [v5.1.3]
+### Changed
+- `workflow-glue` CLI bootstrapping amended to skip imports of all components when not required.
+
 ## [v5.1.2]
 ### Added
 - `xam_ingress` reads `basecall_model` and `runid` to metamap from RG DS tags.

bin/workflow_glue/__init__.py

@@ -3,6 +3,7 @@ import argparse
 import glob
 import importlib
 import os
+import sys
 
 from .util import _log_level, get_main_logger  # noqa: ABS101
 
@@ -11,15 +12,17 @@ __version__ = "0.0.1"
 _package_name = "workflow_glue"
 
 
-def get_components():
+def get_components(allowed_components=None):
     """Find a list of workflow command scripts."""
     logger = get_main_logger(_package_name)
     path = os.path.dirname(os.path.abspath(__file__))
-    components = list()
+    components = dict()
     for fname in glob.glob(os.path.join(path, "*.py")):
         name = os.path.splitext(os.path.basename(fname))[0]
         if name in ("__init__", "util"):
             continue
+        if allowed_components is not None and name not in allowed_components:
+            continue
 
         # leniently attempt to import module
         try:
@@ -34,7 +37,7 @@ def get_components():
         try:
             req = "main", "argparser"
             if all(callable(getattr(mod, x)) for x in req):
-                components.append(name)
+                components[name] = mod
         except Exception:
             pass
     return components
@@ -42,6 +45,8 @@ def get_components():
 
 def cli():
     """Run workflow entry points."""
+    logger = get_main_logger(_package_name)
+    logger.info("Bootstrapping CLI.")
     parser = argparse.ArgumentParser(
         'wf-glue',
         parents=[_log_level()],
@@ -56,16 +61,21 @@ def cli():
         help='additional help', dest='command')
     subparsers.required = True
 
-    # all component demos, plus some others
-    components = [
-        f'{_package_name}.{comp}' for comp in get_components()]
-    for module in components:
-        mod = importlib.import_module(module)
+    # importing everything can take time, try to shortcut
+    if len(sys.argv) > 1:
+        components = get_components(allowed_components=[sys.argv[1]])
+        if not sys.argv[1] in components:
+            logger.warn("Importing all modules, this may take some time.")
+            components = get_components()
+    else:
+        components = get_components()
+
+    # add all module parsers to main CLI
+    for name, module in components.items():
         p = subparsers.add_parser(
-            module.split(".")[-1], parents=[mod.argparser()])
-        p.set_defaults(func=mod.main)
+            name.split(".")[-1], parents=[module.argparser()])
+        p.set_defaults(func=module.main)
 
-    logger = get_main_logger(_package_name)
     args = parser.parse_args()
 
     logger.info("Starting entrypoint.")

nextflow.config

@@ -56,7 +56,7 @@ manifest {
     description     = 'Template workflow'
     mainScript      = 'main.nf'
     nextflowVersion = '>=23.04.2'
-    version         = 'v5.1.2'
+    version         = 'v5.1.3'
 }