From 2524b6a0e02d96aec8ca471208f013e3ddeac0b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Maximilian=20Ke=C3=9Fler?= Date: Sun, 24 Oct 2021 08:35:53 +0000 Subject: [PATCH] Build rework --- .build/header_info.txt | 7 ++ .ci/ci_build.sh | 12 ++++ {build_scripts => .ci/deploy}/__init__.py | 0 .../travis => .ci/deploy}/deploy_message.py | 10 ++- .ci/get_build_repo_from_origin.sh | 18 +++++ .ci/print_deploy_message.py | 7 ++ .gitlab-ci.yml | 71 +++++++++++++++++++ .gitmodules | 4 +- .travis.yml | 13 +--- Makefile | 12 +++- PyTeX | 2 +- build.py | 14 +--- build_scripts/build/__init__.py | 5 -- build_scripts/build/build.py | 67 ----------------- build_scripts/build/build_information.py | 20 ------ build_scripts/build/config.py | 6 -- build_scripts/git_hook/__init__.py | 9 --- build_scripts/git_hook/git_version.py | 53 -------------- build_scripts/git_hook/recent.py | 14 ---- build_scripts/travis/__init__.py | 5 -- 20 files changed, 142 insertions(+), 207 deletions(-) create mode 100644 .build/header_info.txt create mode 100755 .ci/ci_build.sh rename {build_scripts => .ci/deploy}/__init__.py (100%) rename {build_scripts/travis => .ci/deploy}/deploy_message.py (64%) create mode 100755 .ci/get_build_repo_from_origin.sh create mode 100644 .ci/print_deploy_message.py create mode 100644 .gitlab-ci.yml mode change 100644 => 100755 build.py delete mode 100644 build_scripts/build/__init__.py delete mode 100644 build_scripts/build/build.py delete mode 100644 build_scripts/build/build_information.py delete mode 100644 build_scripts/build/config.py delete mode 100644 build_scripts/git_hook/__init__.py delete mode 100644 build_scripts/git_hook/git_version.py delete mode 100644 build_scripts/git_hook/recent.py delete mode 100644 build_scripts/travis/__init__.py diff --git a/.build/header_info.txt b/.build/header_info.txt new file mode 100644 index 0000000..2b6390e --- /dev/null +++ b/.build/header_info.txt @@ -0,0 +1,7 @@ +This LaTeX {latex_file_type} is free software and distributed under the MIT License. You +may use it freely for your purposes. The latest version of the {latex_file_type} can be +obtained via GitHub under + https://github.com/kesslermaximilian/LatexPackages +For further information see the url above. +Reportings of bugs, suggestions and improvements are welcome, see the README +at the Git repository for further information. diff --git a/.ci/ci_build.sh b/.ci/ci_build.sh new file mode 100755 index 0000000..55c43f3 --- /dev/null +++ b/.ci/ci_build.sh @@ -0,0 +1,12 @@ +cd .ci +export COMMIT_MSG=$(python3 print_deploy_message.py) +cd .. +make ci-build +cd build +zip -r LatexPackages.zip LatexPackagesBuild/ -x '*.git*' +tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html +cd LatexPackagesBuild +git add . +git commit -m "${COMMIT_MSG}" || echo "Nothing new to commit" +git push --set-upstream origin ${CI_COMMIT_REF_NAME}-build + diff --git a/build_scripts/__init__.py b/.ci/deploy/__init__.py similarity index 100% rename from build_scripts/__init__.py rename to .ci/deploy/__init__.py diff --git a/build_scripts/travis/deploy_message.py b/.ci/deploy/deploy_message.py similarity index 64% rename from build_scripts/travis/deploy_message.py rename to .ci/deploy/deploy_message.py index 19f0327..042cfcf 100644 --- a/build_scripts/travis/deploy_message.py +++ b/.ci/deploy/deploy_message.py @@ -1,5 +1,11 @@ import os -from build_scripts.git_hook import get_latest_commit + + +def get_latest_commit(repo): + if repo.head.is_detached: + return repo.head.commit + else: + return repo.head.ref.commit def get_deploy_message(repo): @@ -8,6 +14,6 @@ def get_deploy_message(repo): "\n" \ "Build branch {branch} ({hexsha}) from {repo_name}" \ .format(old_msg=old_msg, - branch=os.environ['TRAVIS_BRANCH'], + branch=os.environ['CI_COMMIT_REF_NAME'], hexsha=get_latest_commit(repo).hexsha[0:7], repo_name='kesslermaximilian/LatexPackages') diff --git a/.ci/get_build_repo_from_origin.sh b/.ci/get_build_repo_from_origin.sh new file mode 100755 index 0000000..2fe26aa --- /dev/null +++ b/.ci/get_build_repo_from_origin.sh @@ -0,0 +1,18 @@ +# ! /bin/sh +ssh git@gitlab.com +git clone git@gitlab.com:latexci/packages/LatexPackagesBuild.git build/LatexPackagesBuild +cd build/LatexPackagesBuild +REMOTE_BRANCH=$(git branch -a | sed -n '/remotes\/origin\/.*-build/p' | sed 's/remotes\/origin\///g' | sed 's/-build//g' | sed 's/[[:space:]]//g' | sed -n "/^${CI_COMMIT_REF_NAME}$/p") + +echo ${REMOTE_BRANCH} + +if [ "$REMOTE_BRANCH" = "" ];then + echo "This is the first build on this branch, creating new branch in build repository to push to" + git checkout --orphan ${CI_COMMIT_REF_NAME}-build + ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' | xargs -r git rm --cached + ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' | xargs -r rm -rf +else + echo "Checking out remote branch from last build" + git checkout -b ${REMOTE_BRANCH}-build origin/${REMOTE_BRANCH}-build +fi +cd ../.. diff --git a/.ci/print_deploy_message.py b/.ci/print_deploy_message.py new file mode 100644 index 0000000..6323314 --- /dev/null +++ b/.ci/print_deploy_message.py @@ -0,0 +1,7 @@ +from deploy.deploy_message import get_deploy_message +import git + +if __name__ == "__main__": + repo = git.Repo(search_parent_directories=True) + msg = get_deploy_message(repo) + print(msg) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..9e6036f --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,71 @@ +stages: # List of stages for jobs, and their order of execution + - get + - build + - pages + + +get-build-repo: # This job runs in the build stage, which runs first. + stage: get + before_script: + - apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' + - eval $(ssh-agent -s) + - echo "$GITLAB_DEPLOY_KEY" + - echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null + + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + + - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + + - git config --global user.email "git@maximilian-kessler.de" + - git config --global user.name "Maximilian Keßler (via gitlab runner)" + script: + - echo "Getting old Build repo..." + - .ci/get_build_repo_from_origin.sh + tags: + - latex + artifacts: + paths: + - build/* + +build-packages: # This job runs in the deploy stage. + stage: build # It only runs when *both* jobs in the test stage complete successfully. + before_script: + - apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )' + - eval $(ssh-agent -s) + - echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null + + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + + - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts + - chmod 644 ~/.ssh/known_hosts + + - git config --global user.email "git@maximilian-kessler.de" + - git config --global user.name "Maximilian Keßler (via gitlab runner)" + script: + - echo "Building packages incrementally..." + - sh .ci/ci_build.sh + variables: + GIT_SUBMODULE_STRATEGY: recursive + artifacts: + paths: + - build/* + tags: + - latex + + +pages: + stage: pages + artifacts: + paths: + - public/ + script: + - mv build/ public/ + tags: + - latex + only: + - master diff --git a/.gitmodules b/.gitmodules index e5a2100..8a541a0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "PyTeX"] path = PyTeX - url = https://github.com/kesslermaximilian/PyTeX.git - branch= latex-packages + url = ../PyTeX.git + branch = latex-packages diff --git a/.travis.yml b/.travis.yml index 1fd2351..02b2b62 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,15 +15,8 @@ install: - pip install GitPython script: - - export COMMIT_MSG=$(python3 -c 'from git_version import get_deploy_message; print(get_deploy_message())') - - python3 -c 'from build import build; build("build/LatexPackages/")' - - cd build - - zip -r LatexPackages.zip LatexPackages - - tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html - - cd .. - -after_success: - + - source ./.travis/get_build_repo_from_origin.sh + - source ./.travis/travis_build.sh deploy: - provider: pages @@ -42,7 +35,7 @@ deploy: - provider: pages edge: true github-token: $GITHUB_TOKEN - local-dir: build/LatexPackages + local-dir: build/LatexPackagesBuild repo: kesslermaximilian/LatexPackagesBuild commiter_from_gh: true allow_empty_commit: true diff --git a/Makefile b/Makefile index dcf91a4..e98e350 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,12 @@ .PHONY: build +BUILD_FLAGS=--recursive --git-version --pytex-version --license --author "Maximilian Keßler" --pytex-info-text --extra-header ".build/header_info.txt" --name "prepend-author" +BUILD_DIRS= --source-dir src --build-dir build build: .initsubmodulelock - @python3 build.py + @python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS} + +dirty: .initsubmodulelock + @python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS} --allow-dirty init: .initsubmodulelock .gitconfiglock @@ -21,4 +26,7 @@ config: .gitconfiglock clean: - -rm -r build/ + @-rm -r build/ + +ci-build: + @python3 build.py --source-dir src --build-dir build/LatexPackagesBuild ${BUILD_FLAGS} diff --git a/PyTeX b/PyTeX index 05f53a9..126d420 160000 --- a/PyTeX +++ b/PyTeX @@ -1 +1 @@ -Subproject commit 05f53a917113648a81ab96f353800d64dd9e6ec9 +Subproject commit 126d420b7a6ed2d4b2a42d534ab2e12344869643 diff --git a/build.py b/build.py old mode 100644 new mode 100755 index 256263a..83a03e8 --- a/build.py +++ b/build.py @@ -1,15 +1,7 @@ +#! /usr/bin/python3 import sys -from pathlib import Path -from build_scripts.build import build +from PyTeX.build import parse_and_build if __name__ == "__main__": - check_existence = True - if len(sys.argv) == 2: - if sys.argv[1] == '--only-new': - check_existence = False - build( - src_dir=Path('./src').resolve(), - build_dir=Path('./build').resolve(), - check_existence=check_existence - ) + parse_and_build(sys.argv[1:]) diff --git a/build_scripts/build/__init__.py b/build_scripts/build/__init__.py deleted file mode 100644 index 3b3e07b..0000000 --- a/build_scripts/build/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .build import build - -__all__ = [ - 'build' -] diff --git a/build_scripts/build/build.py b/build_scripts/build/build.py deleted file mode 100644 index 812a3c3..0000000 --- a/build_scripts/build/build.py +++ /dev/null @@ -1,67 +0,0 @@ -from pathlib import Path -import git - -import PyTeX - -from build_scripts.git_hook import get_latest_commit, is_recent - -from .build_information import build_information - - -def build(src_dir: Path, build_dir: Path, check_existence: bool = True): - print('[PyTeX] Getting git repository information...') - extra_header, repo_description = build_information() - print('[PyTeX] Building version {version} of LatexPackages'.format(version=repo_description)) - print('[PyTeX] Latest commit message: ' + get_latest_commit(git.Repo()).message.strip()) - if git.Repo().is_dirty(untracked_files=True): - extra_header += ['WARNING: Local changes to git repository detected.', - ' The build will not be reproducible (!)'] - num_packages = 0 - num_skipped_packages = 0 - num_classes = 0 - num_skipped_classes = 0 - - for file in src_dir.rglob('*.pysty'): - output_dir = build_dir / str(file.parent.relative_to(src_dir)) - if not is_recent(file, git.Repo()): - if not check_existence: - print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name)) - num_skipped_packages += 1 - continue - else: - if (output_dir / ('mkessler-' + str(file.with_suffix('.sty').name))).exists(): - print('[PyTex] Skipping file {file} since it was not modified since ' - 'last build and already exists'.format(file=file.name)) - num_skipped_packages += 1 - continue - - num_packages += 1 - formatter = PyTeX.PackageFormatter( - package_name=file.with_suffix('').name, - extra_header=extra_header) - print('[PyTeX] Writing file {}'.format(formatter.file_name)) - formatter.make_default_macros() - formatter.format_file(file, output_dir) - for file in src_dir.rglob('*.pycls'): - output_dir = build_dir / str(file.parent.relative_to(src_dir)) - if not is_recent(file, git.Repo()): - if not check_existence: - print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name)) - num_skipped_classes += 1 - continue - else: - if (output_dir / ('mkessler-' + str(file.with_suffix('.cls').name))).exists(): - print('[PyTex] Skipping file {file} since it was not modified since ' - 'last build and already exists'.format(file=file.name)) - num_skipped_classes += 1 - continue - output_dir = build_dir / str(file.parent.relative_to(src_dir)) - num_classes += 1 - formatter = PyTeX.ClassFormatter( - class_name=file.with_suffix('').name, - extra_header=extra_header) - print('[PyTeX] Writing class file {}'.format(formatter.file_name)) - formatter.make_default_macros() - formatter.format_file(input_path=file, output_dir=output_dir) - print(f'[PyTeX] Successfully built {num_packages} packages (skipped {num_skipped_packages}) ' - f'and {num_classes} classes (skipped {num_skipped_classes}) in {build_dir}/') diff --git a/build_scripts/build/build_information.py b/build_scripts/build/build_information.py deleted file mode 100644 index 217c3d9..0000000 --- a/build_scripts/build/build_information.py +++ /dev/null @@ -1,20 +0,0 @@ -import git -import datetime - -from build_scripts.git_hook import git_describe, get_latest_commit - -from .config import BUILD_DETAILS - - -def build_information(): - repo = git.Repo() - repo_description = git_describe(get_latest_commit(repo)) - pytex_repo = repo.submodule('PyTeX').module() - pytex_repo_description = git_describe(get_latest_commit(pytex_repo)) - return list(map(lambda line: line.format( - build_time=datetime.datetime.now().strftime('%Y/%m/%d %H:%M'), - pytex_version=pytex_repo_description, - pytex_commit_hash=get_latest_commit(pytex_repo).hexsha[0:7], - packages_version=repo_description, - packages_commit_hash=get_latest_commit(repo).hexsha[0:7] - ), BUILD_DETAILS)), repo_description diff --git a/build_scripts/build/config.py b/build_scripts/build/config.py deleted file mode 100644 index c69f02c..0000000 --- a/build_scripts/build/config.py +++ /dev/null @@ -1,6 +0,0 @@ -BUILD_DETAILS = [ - "Build details:", - " Build time: {build_time}", - " PyTeX version: {pytex_version} (commit {pytex_commit_hash})", - " LatexPackages version: {packages_version} (commit {packages_commit_hash})" -] diff --git a/build_scripts/git_hook/__init__.py b/build_scripts/git_hook/__init__.py deleted file mode 100644 index 851f2bc..0000000 --- a/build_scripts/git_hook/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .git_version import git_describe, get_history, get_latest_commit -from .recent import is_recent - -__all__ = [ - 'git_describe', - 'get_history', - 'get_latest_commit', - 'is_recent' -] diff --git a/build_scripts/git_hook/git_version.py b/build_scripts/git_hook/git_version.py deleted file mode 100644 index ad9f179..0000000 --- a/build_scripts/git_hook/git_version.py +++ /dev/null @@ -1,53 +0,0 @@ -import git -from typing import Dict - - -def get_latest_commit(repo): - if repo.head.is_detached: - return repo.head.commit - else: - return repo.head.ref.commit - - -def get_history(commit: git.objects.commit.Commit, priority=0, depth=0) -> Dict: - commit_history = {commit.hexsha: { - 'priority': priority, - 'depth': depth - }} - try: - if len(commit.parents) > 0: - commit_history.update(get_history(commit.parents[0], priority, depth + 1)) - for parent in commit.parents[1:]: - commit_history.update(get_history(parent, priority + 1, depth + 1)) - except ValueError: - pass - return commit_history - - -def git_describe(commit: git.objects.commit.Commit): - commit_history = get_history(commit) - latest_tag = None - for tag in commit.repo.tags: - sha = tag.commit.hexsha - if sha in commit_history.keys(): - if latest_tag is None: - latest_tag = tag - elif commit_history[sha]['priority'] < commit_history[latest_tag.commit.hexsha]['priority']: - latest_tag = tag - elif commit_history[sha]['priority'] > commit_history[latest_tag.commit.hexsha]['priority']: - pass # move on - elif commit_history[sha]['depth'] < commit_history[latest_tag.commit.hexsha]['depth']: - latest_tag = tag - elif commit_history[sha]['depth'] > commit_history[latest_tag.commit.hexsha]['depth']: - pass # move on - elif tag.object.tagged_date > latest_tag.object.tagged_date: - latest_tag = tag - if latest_tag is None: - return "No tags found - cannot describe anything." - else: - msg = latest_tag.name - if commit_history[latest_tag.commit.hexsha]['depth'] != 0: - msg += "-{depth}".format(depth=commit_history[latest_tag.commit.hexsha]['depth']) - if commit.repo.is_dirty(untracked_files=True): - msg += '-*' - return msg diff --git a/build_scripts/git_hook/recent.py b/build_scripts/git_hook/recent.py deleted file mode 100644 index acc7480..0000000 --- a/build_scripts/git_hook/recent.py +++ /dev/null @@ -1,14 +0,0 @@ -from .git_version import get_latest_commit - - -def is_recent(file, repo): - modified_files = [item.a_path for item in repo.index.diff(None)] - if file in modified_files: - return True - - for parent in get_latest_commit(repo).parents: - newly_committed_files = [item.a_path for item in repo.index.diff(parent)] - if file in newly_committed_files: - return True - return False - diff --git a/build_scripts/travis/__init__.py b/build_scripts/travis/__init__.py deleted file mode 100644 index a756aa3..0000000 --- a/build_scripts/travis/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .deploy_message import get_deploy_message - -__all__ = [ - 'get_deploy_message' -]