Merge branch 'build-rework' into 'master'

Build rework

See merge request latexci/packages/LatexPackages!4

.build/header_info.txt

@@ -0,0 +1,7 @@
+This LaTeX {latex_file_type} is free software and distributed under the MIT License. You
+may use it freely for your purposes. The latest version of the {latex_file_type} can be
+obtained via GitHub under
+   https://github.com/kesslermaximilian/LatexPackages
+For further information see the url above.
+Reportings of bugs, suggestions and improvements are welcome, see the README
+at the Git repository for further information.

.ci/ci_build.sh

@@ -0,0 +1,12 @@
+cd .ci
+export COMMIT_MSG=$(python3 print_deploy_message.py)
+cd ..
+make ci-build
+cd build
+zip -r LatexPackages.zip LatexPackagesBuild/ -x '*.git*'
+tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html
+cd LatexPackagesBuild
+git add .
+git commit -m "${COMMIT_MSG}" || echo "Nothing new to commit"
+git push --set-upstream origin ${CI_COMMIT_REF_NAME}-build
+

.ci/deploy/deploy_message.py

@@ -1,5 +1,11 @@
 import os
-from build_scripts.git_hook import get_latest_commit
+
+
+def get_latest_commit(repo):
+    if repo.head.is_detached:
+        return repo.head.commit
+    else:
+        return repo.head.ref.commit
 
 
 def get_deploy_message(repo):
@@ -8,6 +14,6 @@ def get_deploy_message(repo):
            "\n" \
            "Build branch {branch} ({hexsha}) from {repo_name}" \
         .format(old_msg=old_msg,
-                branch=os.environ['TRAVIS_BRANCH'],
+                branch=os.environ['CI_COMMIT_REF_NAME'],
                 hexsha=get_latest_commit(repo).hexsha[0:7],
                 repo_name='kesslermaximilian/LatexPackages')

.ci/get_build_repo_from_origin.sh

@@ -0,0 +1,18 @@
+# ! /bin/sh
+ssh git@gitlab.com
+git clone git@gitlab.com:latexci/packages/LatexPackagesBuild.git build/LatexPackagesBuild
+cd build/LatexPackagesBuild
+REMOTE_BRANCH=$(git branch -a | sed -n '/remotes\/origin\/.*-build/p' | sed 's/remotes\/origin\///g' | sed 's/-build//g' | sed 's/[[:space:]]//g' | sed -n "/^${CI_COMMIT_REF_NAME}$/p")
+
+echo ${REMOTE_BRANCH}
+
+if [ "$REMOTE_BRANCH" = "" ];then
+    echo "This is the first build on this branch, creating new branch in build repository to push to"
+    git checkout --orphan ${CI_COMMIT_REF_NAME}-build
+    ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' |  xargs -r git rm --cached
+    ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' |  xargs -r rm -rf
+else
+    echo "Checking out remote branch from last build"
+    git checkout -b ${REMOTE_BRANCH}-build origin/${REMOTE_BRANCH}-build
+fi
+cd ../..

.ci/print_deploy_message.py

@@ -0,0 +1,7 @@
+from deploy.deploy_message import get_deploy_message
+import git
+
+if __name__ == "__main__":
+    repo = git.Repo(search_parent_directories=True)
+    msg = get_deploy_message(repo)
+    print(msg)

.gitlab-ci.yml

@@ -0,0 +1,71 @@
+stages:          # List of stages for jobs, and their order of execution
+  - get
+  - build
+  - pages
+
+
+get-build-repo:       # This job runs in the build stage, which runs first.
+  stage: get
+  before_script:
+    - apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$GITLAB_DEPLOY_KEY"
+    - echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
+
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+
+    - git config --global user.email "git@maximilian-kessler.de"
+    - git config --global user.name "Maximilian Keßler (via gitlab runner)"
+  script:
+      - echo "Getting old Build repo..."
+      - .ci/get_build_repo_from_origin.sh
+  tags:
+    - latex
+  artifacts:
+    paths:
+      - build/*
+
+build-packages:      # This job runs in the deploy stage.
+  stage: build  # It only runs when *both* jobs in the test stage complete successfully.
+  before_script:
+    - apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing
+    - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
+    - eval $(ssh-agent -s)
+    - echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
+
+    - mkdir -p ~/.ssh
+    - chmod 700 ~/.ssh
+
+    - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
+    - chmod 644 ~/.ssh/known_hosts
+
+    - git config --global user.email "git@maximilian-kessler.de"
+    - git config --global user.name "Maximilian Keßler (via gitlab runner)"
+  script:
+    - echo "Building packages incrementally..."
+    - sh .ci/ci_build.sh
+  variables:
+      GIT_SUBMODULE_STRATEGY: recursive
+  artifacts:
+    paths:
+        - build/*
+  tags:
+    - latex
+
+
+pages:
+    stage: pages
+    artifacts:
+        paths:
+            - public/
+    script:
+        - mv build/ public/
+    tags:
+        - latex
+    only:
+        - master

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "PyTeX"]
 	path = PyTeX
-    url = https://github.com/kesslermaximilian/PyTeX.git
+    url = ../PyTeX.git
     branch = latex-packages

.travis.yml

@@ -15,15 +15,8 @@ install:
     - pip install GitPython
       
 script:
-    - export COMMIT_MSG=$(python3 -c 'from git_version import get_deploy_message; print(get_deploy_message())')
+    - source ./.travis/get_build_repo_from_origin.sh
-    - python3 -c 'from build import build; build("build/LatexPackages/")'
+    - source ./.travis/travis_build.sh
-    - cd build
-    - zip -r LatexPackages.zip LatexPackages
-    - tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html
-    - cd ..
-
-after_success:
-    
 
 deploy:
     - provider: pages
@@ -42,7 +35,7 @@ deploy:
     - provider: pages
       edge: true
       github-token: $GITHUB_TOKEN
-      local-dir: build/LatexPackages
+      local-dir: build/LatexPackagesBuild
       repo: kesslermaximilian/LatexPackagesBuild
       commiter_from_gh: true
       allow_empty_commit: true

Makefile

@@ -1,7 +1,12 @@
 .PHONY: build
+BUILD_FLAGS=--recursive --git-version --pytex-version --license --author "Maximilian Keßler" --pytex-info-text --extra-header ".build/header_info.txt" --name "prepend-author"
+BUILD_DIRS= --source-dir src --build-dir build
 
 build: .initsubmodulelock
-	@python3 build.py
+	@python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS}
+
+dirty: .initsubmodulelock
+	@python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS} --allow-dirty
 
 init: .initsubmodulelock .gitconfiglock
 
@@ -21,4 +26,7 @@ config: .gitconfiglock
 
 
 clean:
-	-rm -r build/
+	@-rm -r build/
+
+ci-build:
+	@python3 build.py --source-dir src --build-dir build/LatexPackagesBuild ${BUILD_FLAGS}

PyTeX

@@ -1 +1 @@
-Subproject commit 05f53a917113648a81ab96f353800d64dd9e6ec9
+Subproject commit 126d420b7a6ed2d4b2a42d534ab2e12344869643

build.py

@@ -1,15 +1,7 @@
+#! /usr/bin/python3
 import sys
-from pathlib import Path
 
-from build_scripts.build import build
+from PyTeX.build import parse_and_build
 
 if __name__ == "__main__":
-    check_existence = True
+    parse_and_build(sys.argv[1:])
-    if len(sys.argv) == 2:
-        if sys.argv[1] == '--only-new':
-            check_existence = False
-    build(
-        src_dir=Path('./src').resolve(),
-        build_dir=Path('./build').resolve(),
-        check_existence=check_existence
-    )

build_scripts/build/__init__.py

@@ -1,5 +0,0 @@
-from .build import build
-
-__all__ = [
-    'build'
-]

build_scripts/build/build.py

@@ -1,67 +0,0 @@
-from pathlib import Path
-import git
-
-import PyTeX
-
-from build_scripts.git_hook import get_latest_commit, is_recent
-
-from .build_information import build_information
-
-
-def build(src_dir: Path, build_dir: Path, check_existence: bool = True):
-    print('[PyTeX] Getting git repository information...')
-    extra_header, repo_description = build_information()
-    print('[PyTeX] Building version {version} of LatexPackages'.format(version=repo_description))
-    print('[PyTeX] Latest commit message: ' + get_latest_commit(git.Repo()).message.strip())
-    if git.Repo().is_dirty(untracked_files=True):
-        extra_header += ['WARNING: Local changes to git repository detected.',
-                         '         The build will not be reproducible (!)']
-    num_packages = 0
-    num_skipped_packages = 0
-    num_classes = 0
-    num_skipped_classes = 0
-
-    for file in src_dir.rglob('*.pysty'):
-        output_dir = build_dir / str(file.parent.relative_to(src_dir))
-        if not is_recent(file, git.Repo()):
-            if not check_existence:
-                print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name))
-                num_skipped_packages += 1
-                continue
-            else:
-                if (output_dir / ('mkessler-' + str(file.with_suffix('.sty').name))).exists():
-                    print('[PyTex] Skipping file {file} since it was not modified since '
-                          'last build and already exists'.format(file=file.name))
-                    num_skipped_packages += 1
-                    continue
-
-        num_packages += 1
-        formatter = PyTeX.PackageFormatter(
-            package_name=file.with_suffix('').name,
-            extra_header=extra_header)
-        print('[PyTeX] Writing file {}'.format(formatter.file_name))
-        formatter.make_default_macros()
-        formatter.format_file(file, output_dir)
-    for file in src_dir.rglob('*.pycls'):
-        output_dir = build_dir / str(file.parent.relative_to(src_dir))
-        if not is_recent(file, git.Repo()):
-            if not check_existence:
-                print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name))
-                num_skipped_classes += 1
-                continue
-            else:
-                if (output_dir / ('mkessler-' + str(file.with_suffix('.cls').name))).exists():
-                    print('[PyTex] Skipping file {file} since it was not modified since '
-                          'last build and already exists'.format(file=file.name))
-                    num_skipped_classes += 1
-                    continue
-        output_dir = build_dir / str(file.parent.relative_to(src_dir))
-        num_classes += 1
-        formatter = PyTeX.ClassFormatter(
-            class_name=file.with_suffix('').name,
-            extra_header=extra_header)
-        print('[PyTeX] Writing class file {}'.format(formatter.file_name))
-        formatter.make_default_macros()
-        formatter.format_file(input_path=file, output_dir=output_dir)
-    print(f'[PyTeX] Successfully built {num_packages} packages (skipped {num_skipped_packages}) '
-          f'and {num_classes} classes (skipped {num_skipped_classes}) in {build_dir}/')

build_scripts/build/build_information.py

@@ -1,20 +0,0 @@
-import git
-import datetime
-
-from build_scripts.git_hook import git_describe, get_latest_commit
-
-from .config import BUILD_DETAILS
-
-
-def build_information():
-    repo = git.Repo()
-    repo_description = git_describe(get_latest_commit(repo))
-    pytex_repo = repo.submodule('PyTeX').module()
-    pytex_repo_description = git_describe(get_latest_commit(pytex_repo))
-    return list(map(lambda line: line.format(
-        build_time=datetime.datetime.now().strftime('%Y/%m/%d %H:%M'),
-        pytex_version=pytex_repo_description,
-        pytex_commit_hash=get_latest_commit(pytex_repo).hexsha[0:7],
-        packages_version=repo_description,
-        packages_commit_hash=get_latest_commit(repo).hexsha[0:7]
-    ), BUILD_DETAILS)), repo_description

build_scripts/build/config.py

@@ -1,6 +0,0 @@
-BUILD_DETAILS = [
-    "Build details:",
-    "  Build time: {build_time}",
-    "  PyTeX version: {pytex_version} (commit {pytex_commit_hash})",
-    "  LatexPackages version: {packages_version} (commit {packages_commit_hash})"
-]

build_scripts/git_hook/__init__.py

@@ -1,9 +0,0 @@
-from .git_version import git_describe, get_history, get_latest_commit
-from .recent import is_recent
-
-__all__ = [
-    'git_describe',
-    'get_history',
-    'get_latest_commit',
-    'is_recent'
-]

build_scripts/git_hook/git_version.py

@@ -1,53 +0,0 @@
-import git
-from typing import Dict
-
-
-def get_latest_commit(repo):
-    if repo.head.is_detached:
-        return repo.head.commit
-    else:
-        return repo.head.ref.commit
-
-
-def get_history(commit: git.objects.commit.Commit, priority=0, depth=0) -> Dict:
-    commit_history = {commit.hexsha: {
-        'priority': priority,
-        'depth': depth
-    }}
-    try:
-        if len(commit.parents) > 0:
-            commit_history.update(get_history(commit.parents[0], priority, depth + 1))
-            for parent in commit.parents[1:]:
-                commit_history.update(get_history(parent, priority + 1, depth + 1))
-    except ValueError:
-        pass
-    return commit_history
-
-
-def git_describe(commit: git.objects.commit.Commit):
-    commit_history = get_history(commit)
-    latest_tag = None
-    for tag in commit.repo.tags:
-        sha = tag.commit.hexsha
-        if sha in commit_history.keys():
-            if latest_tag is None:
-                latest_tag = tag
-            elif commit_history[sha]['priority'] < commit_history[latest_tag.commit.hexsha]['priority']:
-                latest_tag = tag
-            elif commit_history[sha]['priority'] > commit_history[latest_tag.commit.hexsha]['priority']:
-                pass  # move on
-            elif commit_history[sha]['depth'] < commit_history[latest_tag.commit.hexsha]['depth']:
-                latest_tag = tag
-            elif commit_history[sha]['depth'] > commit_history[latest_tag.commit.hexsha]['depth']:
-                pass  # move on
-            elif tag.object.tagged_date > latest_tag.object.tagged_date:
-                latest_tag = tag
-    if latest_tag is None:
-        return "No tags found - cannot describe anything."
-    else:
-        msg = latest_tag.name
-        if commit_history[latest_tag.commit.hexsha]['depth'] != 0:
-            msg += "-{depth}".format(depth=commit_history[latest_tag.commit.hexsha]['depth'])
-        if commit.repo.is_dirty(untracked_files=True):
-            msg += '-*'
-        return msg

build_scripts/git_hook/recent.py

@@ -1,14 +0,0 @@
-from .git_version import get_latest_commit
-
-
-def is_recent(file, repo):
-    modified_files = [item.a_path for item in repo.index.diff(None)]
-    if file in modified_files:
-        return True
-
-    for parent in get_latest_commit(repo).parents:
-        newly_committed_files = [item.a_path for item in repo.index.diff(parent)]
-        if file in newly_committed_files:
-            return True
-    return False
-

build_scripts/travis/__init__.py

@@ -1,5 +0,0 @@
-from .deploy_message import get_deploy_message
-
-__all__ = [
-    'get_deploy_message'
-]