g2h: Add utility script

Also move the sources into the more conventional src/g2h dir

12 files changed, 367 insertions, 0 deletions

diff --git a/src/g2h/__init__.py b/src/g2h/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/src/g2h/__init__.py
diff --git a/src/g2h/cli.py b/src/g2h/cli.py
new file mode 100644
index 0000000..9eb869c
--- /dev/null
+++ b/src/g2h/cli.py
@@ -0,0 +1,42 @@
+import argparse
+import os
+
+from .git import Git
+from .page_factory import PageFactory
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        prog='g2h',
+        description='Convert a git repository into markdown files Hugo can understand.')
+    # TODO: Defaults for these two values should be $PWD.
+    parser.add_argument('--repo', help='Path to the root of git repository', default=os.getcwd())
+    parser.add_argument('--out', help='Path to where the generated files should be stored', default=os.getcwd())
+    parser.add_argument('--name', help='Name of the project. Default: directory name of repo')
+    parser.add_argument('--content-subdir', help='Subdir(s) where the page will be served from. Default: None', default='')
+
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+    git = Git(args.repo)
+
+    branches = sorted(git.get_branches(), key=lambda b: b.commit.committed_date, reverse=True)
+    commits = git.get_commits()
+    head = git.get_head()
+    # TODO: Fix this for repo=(.|~)
+    project_name = args.name if args.name is not None else os.path.basename(args.repo)
+    tags = sorted(git.get_tags(), key=lambda t: t.commit.committed_date, reverse=True)
+
+    factory = PageFactory(args.out, project_name, args.content_subdir)
+    factory.new_overview(branches, tags, head).render()
+    factory.new_log(commits, branches, tags, head).render()
+    for commit in commits:
+        factory.new_commit(commit).render()
+        tree = factory.new_tree(commit, '', git.get_repo())
+        for tree_page in tree.build_pages(factory):
+            tree_page.render()
+            for file in tree_page.get_files():
+                factory.new_file(commit, file).render()
+
+if __name__ == '__main__':
+    main()
diff --git a/src/g2h/commit_page.py b/src/g2h/commit_page.py
new file mode 100644
index 0000000..84374e6
--- /dev/null
+++ b/src/g2h/commit_page.py
@@ -0,0 +1,36 @@
+import os
+
+from .page import Page
+
+def _redact_local_segment(local_segment):
+    if len(local_segment) == 0:
+        return ''
+    if len(local_segment) < 6:
+        return local_segment[0] + '...'
+    return local_segment[:2] + '...'
+
+def redact_email(email):
+    if email is None or len(email) == 0:
+        return ''
+
+    if '@' not in email:
+        return _redact_local_segment(email)
+
+    local, domain = email.split('@')
+    redacted_local = _redact_local_segment(local)
+
+    return redacted_local + '@' + domain
+
+class CommitPage(Page):
+    def __init__(self, out_dir, name, commit, content_subdir):
+        super(CommitPage, self).__init__('commit.md', out_dir, os.path.join(commit.hexsha, '_index.md'), name, content_subdir)
+        self.commit = commit
+
+    def context(self):
+        return {
+            'commit': self.commit,
+            'title': self.name + ': ' + self.commit.hexsha,
+            'redacted_email': redact_email(self.commit.author.email),
+            'content_subdir': self.content_subdir,
+            'this_commit': self.commit
+        }
diff --git a/src/g2h/file.py b/src/g2h/file.py
new file mode 100644
index 0000000..e07f251
--- /dev/null
+++ b/src/g2h/file.py
@@ -0,0 +1,27 @@
+import os
+
+def generate_breadcrumbs(content_subdir, commit, dir, files):
+    breadcrumbs = '[/](/{})'.format(os.path.join(content_subdir, tree_join(commit, '')))
+    subpath = ''
+    for segment in dir.split(os.sep):
+        if not segment or len(segment) == 0:
+            continue
+        subpath = os.path.join(subpath, sanitize_file_name(segment, files))
+        breadcrumbs += '/[{}](/{})'.format(segment, os.path.join(content_subdir, tree_join(commit, subpath)))
+    return breadcrumbs
+
+
+def sanitize_file_name(file_name, files):
+    if not file_name.startswith('.'):
+        return file_name
+
+    file_names = [ f.name for f in files ]
+    for i in range(1, 10):
+        new_name = file_name.replace('.', '_' * i, 1)
+        if not new_name in file_names:
+            return new_name
+
+    raise RuntimeError('Couldn not find unused sanitized file name')
+
+def tree_join(commit, path):
+    return os.path.join(commit.hexsha, 't', path)
diff --git a/src/g2h/file_page.py b/src/g2h/file_page.py
new file mode 100644
index 0000000..a558254
--- /dev/null
+++ b/src/g2h/file_page.py
@@ -0,0 +1,47 @@
+import base64
+import filetype
+import os
+
+from .file import generate_breadcrumbs, sanitize_file_name, tree_join
+from .page import Page
+from .tree import safe_join
+
+def bytes_to_content(file_bytes):
+    if not file_bytes:
+        return (None, 'empty')
+    try:
+        text = file_bytes.decode('utf-8')
+        return (text, 'text')
+    except UnicodeError:
+        if filetype.is_image(file_bytes):
+            # TODO: Refer to the static file once that's implemented.
+            img = base64.b64encode(file_bytes)
+            return (img.decode('utf-8'), 'image')
+
+    return (None, None)
+
+class FilePage(Page):
+    def __init__(self, out_dir, name, commit, file_name, file_path, content_subdir):
+        tree = safe_join(commit.tree, file_path)
+        super(FilePage, self).__init__('file.md', out_dir, os.path.join(tree_join(commit, file_path), sanitize_file_name(file_name, tree.blobs) + '.md'), name, content_subdir)
+        self.commit = commit
+        self.file_name = file_name
+        self.file_path = file_path
+        self.tree = tree
+
+    def read_file(self):
+        return self.tree.join(self.file_name).data_stream.read()
+
+    def context(self):
+        file_bytes = self.read_file()
+        content, file_type = bytes_to_content(file_bytes)
+
+        return {
+            'breadcrumbs': generate_breadcrumbs(self.content_subdir, self.commit, os.path.join(self.file_path, self.file_name), self.tree.blobs),
+            'content': content,
+            'content_subdir': self.content_subdir,
+            'file_extension': os.path.splitext(self.file_name)[1].lstrip('.'),
+            'title': self.name + ': ' + self.file_name,
+            'this_commit': self.commit,
+            'type': file_type
+        }
diff --git a/src/g2h/git.py b/src/g2h/git.py
new file mode 100644
index 0000000..283135c
--- /dev/null
+++ b/src/g2h/git.py
@@ -0,0 +1,23 @@
+import git
+
+class Git:
+    def __init__(self, repo_path):
+        try:
+            self.repo = git.Repo(repo_path)
+        except git.InvalidGitRepositoryError:
+            raise ValueError(f"Invalid git repository at {repo_path}")
+
+    def get_branches(self):
+        return self.repo.branches
+
+    def get_commits(self):
+        return list(self.repo.iter_commits())
+
+    def get_head(self):
+        return self.repo.head.commit
+
+    def get_repo(self):
+        return self.repo
+
+    def get_tags(self):
+        return self.repo.tags
diff --git a/src/g2h/log_page.py b/src/g2h/log_page.py
new file mode 100644
index 0000000..b8df839
--- /dev/null
+++ b/src/g2h/log_page.py
@@ -0,0 +1,47 @@
+import os
+
+from .page import Page
+
+def ref_names_for_commit(refs, commit_sha):
+    ref_names = []
+    if refs is None:
+        return ref_names
+    for ref in refs:
+        if ref.commit.hexsha == commit_sha:
+            ref_names.append(ref.name)
+
+    return ref_names
+
+class LogPage(Page):
+    def __init__(self, out_dir, name, commits, branches, tags, content_subdir, latest_commit):
+        super(LogPage, self).__init__('log.md', out_dir, 'l.md', name, content_subdir)
+        self.commits = commits
+        self.branches = branches
+        self.tags = tags
+        self.latest_commit = latest_commit
+
+    def _commits_with_refs(self):
+        commits_context = []
+        for commit in self.commits:
+            refs = ref_names_for_commit(self.branches, commit.hexsha) + ref_names_for_commit(self.tags, commit.hexsha)
+            commit_context = {
+                'committed_datetime': commit.committed_datetime,
+                'hexsha': commit.hexsha,
+                'refs': refs,
+                'summary': commit.summary
+            }
+            commits_context.append(commit_context)
+        return commits_context
+
+    def context(self):
+        commits_context = []
+        if self.commits is not None:
+            commits_context = self._commits_with_refs()
+
+        return {
+            'commits': commits_context,
+            'content_subdir': self.content_subdir,
+            'title': self.name + ': Log',
+            'this_commit': self.latest_commit
+        }
+
diff --git a/src/g2h/overview_page.py b/src/g2h/overview_page.py
new file mode 100644
index 0000000..145a8d7
--- /dev/null
+++ b/src/g2h/overview_page.py
@@ -0,0 +1,19 @@
+from .page import Page
+
+class OverviewPage(Page):
+    def __init__(self, out_dir, name, branches, tags, content_subdir, latest_commit):
+        super(OverviewPage, self).__init__('overview.md', out_dir, '_index.md', name, content_subdir)
+        self.branches = branches
+        self.tags = tags
+        self.latest_commit = latest_commit
+
+    def context(self):
+        return {
+            'branches': self.branches,
+            'content_subdir': self.content_subdir,
+            'latest_commit_date': self.latest_commit.committed_datetime,
+            'tags': self.tags,
+            'title': self.name,
+            'this_commit': self.latest_commit
+        }
+
diff --git a/src/g2h/page.py b/src/g2h/page.py
new file mode 100644
index 0000000..afe6ac2
--- /dev/null
+++ b/src/g2h/page.py
@@ -0,0 +1,21 @@
+from jinja2 import Environment, FileSystemLoader
+import os
+
+class Page:
+    def __init__(self, template_file, out_dir, out_file, name, content_subdir):
+        self.out_file_path = os.path.join(out_dir, out_file)
+        self.name = name
+        self.content_subdir = content_subdir
+        template_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'templates'))
+        self.template = Environment(loader=FileSystemLoader(template_dir)).get_template(template_file)
+
+    def context(self):
+        # TODO: Probably better to delegate to a method here so name and subdir don't have to be
+        # passed around as much.
+        raise NotImplementedError("Must override context")
+
+    def render(self):
+        content = self.template.render(self.context())
+        os.makedirs(os.path.dirname(self.out_file_path), exist_ok=True)
+        with open(self.out_file_path, 'w', encoding='utf-8') as f:
+            f.write(content)
diff --git a/src/g2h/page_factory.py b/src/g2h/page_factory.py
new file mode 100644
index 0000000..4a46f14
--- /dev/null
+++ b/src/g2h/page_factory.py
@@ -0,0 +1,30 @@
+import os
+
+from .commit_page import CommitPage
+from .file_page import FilePage
+from .overview_page import OverviewPage
+from .log_page import LogPage
+from .tree_page import TreePage
+
+class PageFactory:
+    def __init__(self, out_dir, name, content_subdir):
+        self.out_dir = out_dir
+        self.name = name
+        self.content_subdir = content_subdir
+
+    def new_commit(self, commit):
+        return CommitPage(self.out_dir, self.name, commit, self.content_subdir)
+
+    def new_file(self, commit, path):
+        filename = os.path.basename(path)
+        dir = os.path.dirname(path)
+        return FilePage(self.out_dir, self.name, commit, filename, dir, self.content_subdir)
+
+    def new_overview(self, branches, tags, head):
+        return OverviewPage(self.out_dir, self.name, branches, tags, self.content_subdir, head)
+
+    def new_log(self, commits, branches, tags, head):
+        return LogPage(self.out_dir, self.name, commits, branches, tags, self.content_subdir, head)
+
+    def new_tree(self, commit, tree_segment, repo):
+        return TreePage(self.out_dir, self.name, commit, tree_segment, repo, self.content_subdir)
diff --git a/src/g2h/tree.py b/src/g2h/tree.py
new file mode 100644
index 0000000..bd8116c
--- /dev/null
+++ b/src/g2h/tree.py
@@ -0,0 +1,2 @@
+def safe_join(tree, path):
+    return tree.join(path) if path else tree
diff --git a/src/g2h/tree_page.py b/src/g2h/tree_page.py
new file mode 100644
index 0000000..df65a33
--- /dev/null
+++ b/src/g2h/tree_page.py
@@ -0,0 +1,73 @@
+import os
+
+from .file import generate_breadcrumbs, sanitize_file_name, tree_join
+from .page import Page
+from .tree import safe_join
+
+class TreePage(Page):
+    def __init__(self, out_dir, name, commit, sub_tree, repo, content_subdir):
+        tree = safe_join(commit.tree, sub_tree)
+        super(TreePage, self).__init__('tree.md', out_dir, os.path.join(tree_join(commit, tree.path), '_index.md'), name, content_subdir)
+        self.tree = tree
+        self.repo = repo
+        self.commit = commit
+        self.sub_tree = sub_tree
+
+    def _generate_entries(self, nodes):
+        entries = []
+        for node in nodes:
+            latest_commit = next(self.repo.iter_commits(self.commit, max_count=1, paths=node.path))
+            name = node.name
+            file_name = name
+            if node.type == 'tree':
+                name += '/'
+                file_name = name
+            else:
+                file_name = sanitize_file_name(name, self.tree.blobs)
+            entry = {
+                'file_name': file_name,
+                'name': name,
+                'modified': latest_commit.committed_datetime
+            }
+            entries.append(entry)
+        return entries
+
+    def _get_entries(self):
+        files = self._generate_entries(self.tree.blobs)
+        dirs = self._generate_entries(self.tree.trees)
+        files.sort(key=lambda x: x['name'])
+        dirs.sort(key=lambda x: x['name'])
+        return dirs + files
+
+    def context(self):
+        return {
+            'breadcrumbs': generate_breadcrumbs(self.content_subdir, self.commit, self.sub_tree, self.tree.blobs),
+            'content_subdir': self.content_subdir,
+            'entries': self._get_entries(),
+            # TODO: Make this fit the rest of the system.
+            'subdir': '/'.join([self.content_subdir, tree_join(self.commit, self.tree.path)]),
+            'this_commit': self.commit,
+            'title': self.name + ': Tree'
+        }
+
+    def build_pages(self, factory):
+        visited = set()
+        remaining = [self.sub_tree]
+        tree_pages = []
+        while remaining:
+            current = remaining.pop()
+            if not current in visited:
+                visited.add(current)
+                tree = safe_join(self.commit.tree, current)
+                tree_pages.append(factory.new_tree(self.commit, tree.path, self.repo))
+                for sub_tree in tree.trees:
+                    remaining.append(sub_tree.path)
+
+        return tree_pages
+
+    def get_files(self):
+        file_paths = []
+        for blob in self.tree.blobs:
+            file_paths.append(blob.path)
+
+        return file_paths