Use a Git submodule fetcher to replace the unstable automatic submodule scan

parent b497b688
......@@ -161,12 +161,6 @@ def _get_parser():
version=parser.prog +
" " +
__version__)
parser.add_argument(
'-m',
'--submodule',
action='store_true',
dest="submodule",
help="enable automatic Git submodule scanning")
parser.add_argument(
'-a',
'--all',
......
......@@ -30,7 +30,6 @@ import sys
from hdlmake.tools import load_syn_tool, load_sim_tool
from hdlmake.util import shell
from hdlmake.util import path as path_mod
from hdlmake.util.termcolor import colored
from hdlmake import new_dep_solver as dep_solver
from hdlmake.srcfile import SourceFileSet, VHDLFile, VerilogFile, SVFile
......@@ -107,7 +106,7 @@ class Action(list):
from hdlmake.module import Module, ModuleArgs
self._deps_solved = False
new_module_args = ModuleArgs()
new_module_args.set_args(parent, path_mod.relpath(url), source, fetchto)
new_module_args.set_args(parent, url, source, fetchto)
new_module = Module(new_module_args, self)
if not self.__contains(new_module):
self._add(new_module)
......
......@@ -31,8 +31,8 @@ import os.path
import hdlmake.fetch as fetch
import hdlmake.new_dep_solver as dep_solver
from hdlmake.util import path as path_mod
from hdlmake.fetch import Svn, Git, Local
from hdlmake.fetch import SVN, GIT, LOCAL
from hdlmake.fetch import Svn, Git, GitSM, Local
from hdlmake.fetch import SVN, GIT, GITSM, LOCAL
from .action import Action
......@@ -43,6 +43,7 @@ class ActionCore(Action):
def __init__(self, *args):
super(ActionCore, self).__init__(*args)
self.git_backend = Git()
self.gitsm_backend = GitSM()
self.svn_backend = Svn()
self.local_backend = Local()
......@@ -79,6 +80,8 @@ class ActionCore(Action):
result = self.svn_backend.fetch(module)
elif module.source is GIT:
result = self.git_backend.fetch(module)
elif module.source is GITSM:
result = self.gitsm_backend.fetch(module)
elif module.source is LOCAL:
result = self.local_backend.fetch(module)
if result is False:
......@@ -88,6 +91,7 @@ class ActionCore(Action):
new_modules.extend(module.local)
new_modules.extend(module.svn)
new_modules.extend(module.git)
new_modules.extend(module.gitsm)
return new_modules
fetch_queue = [m for m in self]
......@@ -113,12 +117,12 @@ class ActionCore(Action):
"""Fetch the missing required modules from their remote origin"""
logging.info("Fetching needed modules.")
for mod in self:
if mod.isfetched:
if mod.isfetched and not mod.manifest_dict == None:
if 'fetch_pre_cmd' in mod.manifest_dict:
os.system(mod.manifest_dict.get("fetch_pre_cmd", ''))
self._fetch_all()
for mod in self:
if mod.isfetched:
if mod.isfetched and not mod.manifest_dict == None:
if 'fetch_post_cmd' in mod.manifest_dict:
os.system(mod.manifest_dict.get("fetch_post_cmd", ''))
logging.info("All modules fetched.")
......@@ -127,7 +131,7 @@ class ActionCore(Action):
"""Delete the local copy of the fetched modules"""
logging.info("Removing fetched modules..")
remove_list = [mod_aux for mod_aux in self
if mod_aux.source in [fetch.GIT, fetch.SVN]
if mod_aux.source in [fetch.GIT, fetch.GITSM, fetch.SVN]
and mod_aux.isfetched]
remove_list.reverse() # we will remove modules in backward order
if len(remove_list):
......@@ -181,6 +185,8 @@ class ActionCore(Action):
"""Private function that returns a string with the source type"""
if source_code == fetch.GIT:
return "git"
elif source_code == fetch.GITSM:
return "gitsm"
elif source_code == fetch.SVN:
return "svn"
elif source_code == fetch.LOCAL:
......@@ -192,10 +198,11 @@ class ActionCore(Action):
self._print_comment("# MODULE UNFETCHED! -> %s" % mod_aux.url)
else:
self._print_comment("# MODULE START -> %s" % mod_aux.url)
if mod_aux.source in [fetch.SVN, fetch.GIT]:
if mod_aux.source in [fetch.SVN, fetch.GIT, fetch.GITSM]:
self._print_comment("# * URL: " + mod_aux.url)
if (mod_aux.source in [fetch.SVN, fetch.GIT, fetch.LOCAL] and
mod_aux.parent):
if (mod_aux.source
in [fetch.SVN, fetch.GIT, fetch.GITSM, fetch.LOCAL]
and mod_aux.parent):
self._print_comment("# * The parent for this module is: %s"
% mod_aux.parent.url)
else:
......
......@@ -21,7 +21,7 @@
"""This package provides stuff to handle local and remote repositories"""
from .constants import (GIT, SVN, LOCAL)
from .git import Git
from .constants import (GIT, GITSM, SVN, LOCAL)
from .git import Git, GitSM
from .svn import Svn
from .local import Local
......@@ -26,5 +26,6 @@
"""Module providing the constants required for the fetch process"""
GIT = 1
SVN = 2
LOCAL = 3
GITSM = 2
SVN = 3
LOCAL = 4
......@@ -36,7 +36,7 @@ class Git(Fetcher):
used to fetch and handle Git repositories"""
def __init__(self):
pass
self.submodule = False
@staticmethod
def get_git_toplevel(module):
......@@ -55,8 +55,11 @@ class Git(Fetcher):
"""Get the commit for a repository if defined in Git submodules"""
status_line = shell.run("git submodule status %s" % submodule_dir)
status_line = status_line.split()
if len(status_line) == 2:
if len(status_line) == 2 or len(status_line) == 3:
if status_line[0][0] in ['-', '+', 'U']:
return status_line[0][1:]
else:
return status_line[0]
else:
return None
......@@ -93,6 +96,12 @@ class Git(Fetcher):
cmd = cmd.format(mod_path, checkout_id)
if os.system(cmd) != 0:
return False
if self.submodule and not module.isfetched:
cmd = ("(cd {0} && git submodule init &&"
"git submodule update --recursive)")
cmd = cmd.format(mod_path)
if os.system(cmd) != 0:
return False
module.isfetched = True
module.path = mod_path
return True
......@@ -146,3 +155,9 @@ class Git(Fetcher):
finally:
os.chdir(cur_dir)
return config_submodules
class GitSM(Git):
def __init__(self):
self.submodule = True
......@@ -77,6 +77,7 @@ class ManifestParser(ConfigParser):
self.add_type('files', type_new=[])
self.add_allowed_key('modules', key="svn")
self.add_allowed_key('modules', key="git")
self.add_allowed_key('modules', key="gitsm")
self.add_allowed_key('modules', key="local")
fetch_options = [
{'name': 'fetchto',
......
......@@ -45,6 +45,7 @@ class ModuleContent(ModuleCore):
# Manifest Modules Properties
self.local = []
self.git = []
self.gitsm = []
self.svn = []
self.incl_makefiles = []
super(ModuleContent, self).__init__()
......@@ -135,8 +136,23 @@ class ModuleContent(ModuleCore):
else:
self.git = []
if "gitsm" in self.manifest_dict["modules"]:
self.manifest_dict["modules"]["gitsm"] = path_mod.flatten_list(
self.manifest_dict["modules"]["gitsm"])
gitsm_mods = []
for url in self.manifest_dict["modules"]["gitsm"]:
gitsm_mods.append(self.pool.new_module(parent=self,
url=url,
source=fetch.GITSM,
fetchto=fetchto))
self.gitsm = gitsm_mods
else:
self.gitsm = []
def process_git_submodules(self):
"""Get the submodules if found in the Manifest path"""
if not self.source == fetch.GITSM:
return
git_submodule_dict = fetch.Git.get_git_submodules(self)
git_toplevel = fetch.Git.get_git_toplevel(self)
for submodule_key in git_submodule_dict.keys():
......
......@@ -55,6 +55,7 @@ class ModuleConfig(object):
basename = self.basename()
path = path_mod.relpath(os.path.abspath(
os.path.join(fetchto, basename)))
# Check if the module dir exists and is not empty
if os.path.exists(path) and os.listdir(path):
self.path = path
......@@ -74,7 +75,7 @@ class ModuleConfig(object):
"Path to the local module doesn't exist:\n" + url
+ "\nThis module was instantiated in: " + str(self.parent))
quit()
self.path = url
self.path = path_mod.relpath(url)
self.isfetched = True
def _check_filepath(self, filepath):
......
......@@ -73,7 +73,7 @@ class Module(ModuleContent):
else:
return submodule_list
return __nonull(self.local) + __nonull(self.git) \
+ __nonull(self.svn)
+ __nonull(self.gitsm) + __nonull(self.svn)
def remove_dir_from_disk(self):
"""Delete the module dir if it is already fetched and available"""
......@@ -176,7 +176,6 @@ PARSE START: %s
# Process the parsed manifest_dict to assign the module properties
self.process_manifest()
if self.pool.options.submodule:
self.process_git_submodules()
# Parse every detected submodule
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment