Skip to content

H
Hdlmake
Commit c0fdb014 authored by Javier D. Garcia-Lasheras's avatar Javier D. Garcia-Lasheras
Browse files
Options

Use a Git submodule fetcher to replace the unstable automatic submodule scan

parent b497b688
Show whitespace changes
Inline Side-by-side
Showing with 60 additions and 27 deletions
hdlmake/__main__.py
View file @ c0fdb014
...@@ -161,12 +161,6 @@ def _get_parser(): ...@@ -161,12 +161,6 @@ def _get_parser():
version=parser.prog + version=parser.prog +
" " + " " +
__version__) __version__)
parser.add_argument(
'-m',
'--submodule',
action='store_true',
dest="submodule",
help="enable automatic Git submodule scanning")
parser.add_argument( parser.add_argument(
'-a', '-a',
'--all', '--all',
......
hdlmake/action/action.py
View file @ c0fdb014
...@@ -30,7 +30,6 @@ import sys ...@@ -30,7 +30,6 @@ import sys
from hdlmake.tools import load_syn_tool, load_sim_tool from hdlmake.tools import load_syn_tool, load_sim_tool
from hdlmake.util import shell from hdlmake.util import shell
from hdlmake.util import path as path_mod
from hdlmake.util.termcolor import colored from hdlmake.util.termcolor import colored
from hdlmake import new_dep_solver as dep_solver from hdlmake import new_dep_solver as dep_solver
from hdlmake.srcfile import SourceFileSet, VHDLFile, VerilogFile, SVFile from hdlmake.srcfile import SourceFileSet, VHDLFile, VerilogFile, SVFile
...@@ -107,7 +106,7 @@ class Action(list): ...@@ -107,7 +106,7 @@ class Action(list):
from hdlmake.module import Module, ModuleArgs from hdlmake.module import Module, ModuleArgs
self._deps_solved = False self._deps_solved = False
new_module_args = ModuleArgs() new_module_args = ModuleArgs()
new_module_args.set_args(parent, path_mod.relpath(url), source, fetchto) new_module_args.set_args(parent, url, source, fetchto)
new_module = Module(new_module_args, self) new_module = Module(new_module_args, self)
if not self.__contains(new_module): if not self.__contains(new_module):
self._add(new_module) self._add(new_module)
......
hdlmake/action/core.py
View file @ c0fdb014
...@@ -31,8 +31,8 @@ import os.path ...@@ -31,8 +31,8 @@ import os.path
import hdlmake.fetch as fetch import hdlmake.fetch as fetch
import hdlmake.new_dep_solver as dep_solver import hdlmake.new_dep_solver as dep_solver
from hdlmake.util import path as path_mod from hdlmake.util import path as path_mod
from hdlmake.fetch import Svn, Git, Local from hdlmake.fetch import Svn, Git, GitSM, Local
from hdlmake.fetch import SVN, GIT, LOCAL from hdlmake.fetch import SVN, GIT, GITSM, LOCAL
from .action import Action from .action import Action
...@@ -43,6 +43,7 @@ class ActionCore(Action): ...@@ -43,6 +43,7 @@ class ActionCore(Action):
def __init__(self, *args): def __init__(self, *args):
super(ActionCore, self).__init__(*args) super(ActionCore, self).__init__(*args)
self.git_backend = Git() self.git_backend = Git()
self.gitsm_backend = GitSM()
self.svn_backend = Svn() self.svn_backend = Svn()
self.local_backend = Local() self.local_backend = Local()
...@@ -79,6 +80,8 @@ class ActionCore(Action): ...@@ -79,6 +80,8 @@ class ActionCore(Action):
result = self.svn_backend.fetch(module) result = self.svn_backend.fetch(module)
elif module.source is GIT: elif module.source is GIT:
result = self.git_backend.fetch(module) result = self.git_backend.fetch(module)
elif module.source is GITSM:
result = self.gitsm_backend.fetch(module)
elif module.source is LOCAL: elif module.source is LOCAL:
result = self.local_backend.fetch(module) result = self.local_backend.fetch(module)
if result is False: if result is False:
...@@ -88,6 +91,7 @@ class ActionCore(Action): ...@@ -88,6 +91,7 @@ class ActionCore(Action):
new_modules.extend(module.local) new_modules.extend(module.local)
new_modules.extend(module.svn) new_modules.extend(module.svn)
new_modules.extend(module.git) new_modules.extend(module.git)
new_modules.extend(module.gitsm)
return new_modules return new_modules
fetch_queue = [m for m in self] fetch_queue = [m for m in self]
...@@ -113,12 +117,12 @@ class ActionCore(Action): ...@@ -113,12 +117,12 @@ class ActionCore(Action):
"""Fetch the missing required modules from their remote origin""" """Fetch the missing required modules from their remote origin"""
logging.info("Fetching needed modules.") logging.info("Fetching needed modules.")
for mod in self: for mod in self:
if mod.isfetched: if mod.isfetched and not mod.manifest_dict == None:
if 'fetch_pre_cmd' in mod.manifest_dict: if 'fetch_pre_cmd' in mod.manifest_dict:
os.system(mod.manifest_dict.get("fetch_pre_cmd", '')) os.system(mod.manifest_dict.get("fetch_pre_cmd", ''))
self._fetch_all() self._fetch_all()
for mod in self: for mod in self:
if mod.isfetched: if mod.isfetched and not mod.manifest_dict == None:
if 'fetch_post_cmd' in mod.manifest_dict: if 'fetch_post_cmd' in mod.manifest_dict:
os.system(mod.manifest_dict.get("fetch_post_cmd", '')) os.system(mod.manifest_dict.get("fetch_post_cmd", ''))
logging.info("All modules fetched.") logging.info("All modules fetched.")
...@@ -127,7 +131,7 @@ class ActionCore(Action): ...@@ -127,7 +131,7 @@ class ActionCore(Action):
"""Delete the local copy of the fetched modules""" """Delete the local copy of the fetched modules"""
logging.info("Removing fetched modules..") logging.info("Removing fetched modules..")
remove_list = [mod_aux for mod_aux in self remove_list = [mod_aux for mod_aux in self
if mod_aux.source in [fetch.GIT, fetch.SVN] if mod_aux.source in [fetch.GIT, fetch.GITSM, fetch.SVN]
and mod_aux.isfetched] and mod_aux.isfetched]
remove_list.reverse() # we will remove modules in backward order remove_list.reverse() # we will remove modules in backward order
if len(remove_list): if len(remove_list):
...@@ -181,6 +185,8 @@ class ActionCore(Action): ...@@ -181,6 +185,8 @@ class ActionCore(Action):
"""Private function that returns a string with the source type""" """Private function that returns a string with the source type"""
if source_code == fetch.GIT: if source_code == fetch.GIT:
return "git" return "git"
elif source_code == fetch.GITSM:
return "gitsm"
elif source_code == fetch.SVN: elif source_code == fetch.SVN:
return "svn" return "svn"
elif source_code == fetch.LOCAL: elif source_code == fetch.LOCAL:
...@@ -192,10 +198,11 @@ class ActionCore(Action): ...@@ -192,10 +198,11 @@ class ActionCore(Action):
self._print_comment("# MODULE UNFETCHED! -> %s" % mod_aux.url) self._print_comment("# MODULE UNFETCHED! -> %s" % mod_aux.url)
else: else:
self._print_comment("# MODULE START -> %s" % mod_aux.url) self._print_comment("# MODULE START -> %s" % mod_aux.url)
if mod_aux.source in [fetch.SVN, fetch.GIT]: if mod_aux.source in [fetch.SVN, fetch.GIT, fetch.GITSM]:
self._print_comment("# * URL: " + mod_aux.url) self._print_comment("# * URL: " + mod_aux.url)
if (mod_aux.source in [fetch.SVN, fetch.GIT, fetch.LOCAL] and if (mod_aux.source
mod_aux.parent): in [fetch.SVN, fetch.GIT, fetch.GITSM, fetch.LOCAL]
and mod_aux.parent):
self._print_comment("# * The parent for this module is: %s" self._print_comment("# * The parent for this module is: %s"
% mod_aux.parent.url) % mod_aux.parent.url)
else: else:
......
hdlmake/fetch/__init__.py
View file @ c0fdb014
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
"""This package provides stuff to handle local and remote repositories""" """This package provides stuff to handle local and remote repositories"""
from .constants import (GIT, SVN, LOCAL) from .constants import (GIT, GITSM, SVN, LOCAL)
from .git import Git from .git import Git, GitSM
from .svn import Svn from .svn import Svn
from .local import Local from .local import Local
hdlmake/fetch/constants.py
View file @ c0fdb014
...@@ -26,5 +26,6 @@ ...@@ -26,5 +26,6 @@
"""Module providing the constants required for the fetch process""" """Module providing the constants required for the fetch process"""
GIT = 1 GIT = 1
SVN = 2 GITSM = 2
LOCAL = 3 SVN = 3
LOCAL = 4
hdlmake/fetch/git.py
View file @ c0fdb014
...@@ -36,7 +36,7 @@ class Git(Fetcher): ...@@ -36,7 +36,7 @@ class Git(Fetcher):
used to fetch and handle Git repositories""" used to fetch and handle Git repositories"""
def __init__(self): def __init__(self):
pass self.submodule = False
@staticmethod @staticmethod
def get_git_toplevel(module): def get_git_toplevel(module):
...@@ -55,8 +55,11 @@ class Git(Fetcher): ...@@ -55,8 +55,11 @@ class Git(Fetcher):
"""Get the commit for a repository if defined in Git submodules""" """Get the commit for a repository if defined in Git submodules"""
status_line = shell.run("git submodule status %s" % submodule_dir) status_line = shell.run("git submodule status %s" % submodule_dir)
status_line = status_line.split() status_line = status_line.split()
if len(status_line) == 2: if len(status_line) == 2 or len(status_line) == 3:
if status_line[0][0] in ['-', '+', 'U']:
return status_line[0][1:] return status_line[0][1:]
else:
return status_line[0]
else: else:
return None return None
...@@ -93,6 +96,12 @@ class Git(Fetcher): ...@@ -93,6 +96,12 @@ class Git(Fetcher):
cmd = cmd.format(mod_path, checkout_id) cmd = cmd.format(mod_path, checkout_id)
if os.system(cmd) != 0: if os.system(cmd) != 0:
return False return False
if self.submodule and not module.isfetched:
cmd = ("(cd {0} && git submodule init &&"
"git submodule update --recursive)")
cmd = cmd.format(mod_path)
if os.system(cmd) != 0:
return False
module.isfetched = True module.isfetched = True
module.path = mod_path module.path = mod_path
return True return True
...@@ -146,3 +155,9 @@ class Git(Fetcher): ...@@ -146,3 +155,9 @@ class Git(Fetcher):
finally: finally:
os.chdir(cur_dir) os.chdir(cur_dir)
return config_submodules return config_submodules
class GitSM(Git):
def __init__(self):
self.submodule = True
hdlmake/manifest_parser/variables.py
View file @ c0fdb014
...@@ -77,6 +77,7 @@ class ManifestParser(ConfigParser): ...@@ -77,6 +77,7 @@ class ManifestParser(ConfigParser):
self.add_type('files', type_new=[]) self.add_type('files', type_new=[])
self.add_allowed_key('modules', key="svn") self.add_allowed_key('modules', key="svn")
self.add_allowed_key('modules', key="git") self.add_allowed_key('modules', key="git")
self.add_allowed_key('modules', key="gitsm")
self.add_allowed_key('modules', key="local") self.add_allowed_key('modules', key="local")
fetch_options = [ fetch_options = [
{'name': 'fetchto', {'name': 'fetchto',
......
hdlmake/module/content.py
View file @ c0fdb014
...@@ -45,6 +45,7 @@ class ModuleContent(ModuleCore): ...@@ -45,6 +45,7 @@ class ModuleContent(ModuleCore):
# Manifest Modules Properties # Manifest Modules Properties
self.local = [] self.local = []
self.git = [] self.git = []
self.gitsm = []
self.svn = [] self.svn = []
self.incl_makefiles = [] self.incl_makefiles = []
super(ModuleContent, self).__init__() super(ModuleContent, self).__init__()
...@@ -135,8 +136,23 @@ class ModuleContent(ModuleCore): ...@@ -135,8 +136,23 @@ class ModuleContent(ModuleCore):
else: else:
self.git = [] self.git = []
if "gitsm" in self.manifest_dict["modules"]:
self.manifest_dict["modules"]["gitsm"] = path_mod.flatten_list(
self.manifest_dict["modules"]["gitsm"])
gitsm_mods = []
for url in self.manifest_dict["modules"]["gitsm"]:
gitsm_mods.append(self.pool.new_module(parent=self,
url=url,
source=fetch.GITSM,
fetchto=fetchto))
self.gitsm = gitsm_mods
else:
self.gitsm = []
def process_git_submodules(self): def process_git_submodules(self):
"""Get the submodules if found in the Manifest path""" """Get the submodules if found in the Manifest path"""
if not self.source == fetch.GITSM:
return
git_submodule_dict = fetch.Git.get_git_submodules(self) git_submodule_dict = fetch.Git.get_git_submodules(self)
git_toplevel = fetch.Git.get_git_toplevel(self) git_toplevel = fetch.Git.get_git_toplevel(self)
for submodule_key in git_submodule_dict.keys(): for submodule_key in git_submodule_dict.keys():
......
hdlmake/module/core.py
View file @ c0fdb014
...@@ -55,6 +55,7 @@ class ModuleConfig(object): ...@@ -55,6 +55,7 @@ class ModuleConfig(object):
basename = self.basename() basename = self.basename()
path = path_mod.relpath(os.path.abspath( path = path_mod.relpath(os.path.abspath(
os.path.join(fetchto, basename))) os.path.join(fetchto, basename)))
# Check if the module dir exists and is not empty # Check if the module dir exists and is not empty
if os.path.exists(path) and os.listdir(path): if os.path.exists(path) and os.listdir(path):
self.path = path self.path = path
...@@ -74,7 +75,7 @@ class ModuleConfig(object): ...@@ -74,7 +75,7 @@ class ModuleConfig(object):
"Path to the local module doesn't exist:\n" + url "Path to the local module doesn't exist:\n" + url
+ "\nThis module was instantiated in: " + str(self.parent)) + "\nThis module was instantiated in: " + str(self.parent))
quit() quit()
self.path = url self.path = path_mod.relpath(url)
self.isfetched = True self.isfetched = True
def _check_filepath(self, filepath): def _check_filepath(self, filepath):
......
hdlmake/module/module.py
View file @ c0fdb014
...@@ -73,7 +73,7 @@ class Module(ModuleContent): ...@@ -73,7 +73,7 @@ class Module(ModuleContent):
else: else:
return submodule_list return submodule_list
return __nonull(self.local) + __nonull(self.git) \ return __nonull(self.local) + __nonull(self.git) \
+ __nonull(self.svn) + __nonull(self.gitsm) + __nonull(self.svn)
def remove_dir_from_disk(self): def remove_dir_from_disk(self):
"""Delete the module dir if it is already fetched and available""" """Delete the module dir if it is already fetched and available"""
...@@ -176,7 +176,6 @@ PARSE START: %s ...@@ -176,7 +176,6 @@ PARSE START: %s
# Process the parsed manifest_dict to assign the module properties # Process the parsed manifest_dict to assign the module properties
self.process_manifest() self.process_manifest()
if self.pool.options.submodule:
self.process_git_submodules() self.process_git_submodules()
# Parse every detected submodule # Parse every detected submodule
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment