Merge branch 'build-rework' into 'master'

Build rework

See merge request latexci/packages/LatexPackages!4
This commit is contained in:
Maximilian Keßler 2021-10-24 08:35:54 +00:00
commit 0e520739a3
20 changed files with 142 additions and 207 deletions

7
.build/header_info.txt Normal file
View file

@ -0,0 +1,7 @@
This LaTeX {latex_file_type} is free software and distributed under the MIT License. You
may use it freely for your purposes. The latest version of the {latex_file_type} can be
obtained via GitHub under
https://github.com/kesslermaximilian/LatexPackages
For further information see the url above.
Reportings of bugs, suggestions and improvements are welcome, see the README
at the Git repository for further information.

12
.ci/ci_build.sh Executable file
View file

@ -0,0 +1,12 @@
cd .ci
export COMMIT_MSG=$(python3 print_deploy_message.py)
cd ..
make ci-build
cd build
zip -r LatexPackages.zip LatexPackagesBuild/ -x '*.git*'
tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html
cd LatexPackagesBuild
git add .
git commit -m "${COMMIT_MSG}" || echo "Nothing new to commit"
git push --set-upstream origin ${CI_COMMIT_REF_NAME}-build

View file

@ -1,5 +1,11 @@
import os import os
from build_scripts.git_hook import get_latest_commit
def get_latest_commit(repo):
if repo.head.is_detached:
return repo.head.commit
else:
return repo.head.ref.commit
def get_deploy_message(repo): def get_deploy_message(repo):
@ -8,6 +14,6 @@ def get_deploy_message(repo):
"\n" \ "\n" \
"Build branch {branch} ({hexsha}) from {repo_name}" \ "Build branch {branch} ({hexsha}) from {repo_name}" \
.format(old_msg=old_msg, .format(old_msg=old_msg,
branch=os.environ['TRAVIS_BRANCH'], branch=os.environ['CI_COMMIT_REF_NAME'],
hexsha=get_latest_commit(repo).hexsha[0:7], hexsha=get_latest_commit(repo).hexsha[0:7],
repo_name='kesslermaximilian/LatexPackages') repo_name='kesslermaximilian/LatexPackages')

View file

@ -0,0 +1,18 @@
# ! /bin/sh
ssh git@gitlab.com
git clone git@gitlab.com:latexci/packages/LatexPackagesBuild.git build/LatexPackagesBuild
cd build/LatexPackagesBuild
REMOTE_BRANCH=$(git branch -a | sed -n '/remotes\/origin\/.*-build/p' | sed 's/remotes\/origin\///g' | sed 's/-build//g' | sed 's/[[:space:]]//g' | sed -n "/^${CI_COMMIT_REF_NAME}$/p")
echo ${REMOTE_BRANCH}
if [ "$REMOTE_BRANCH" = "" ];then
echo "This is the first build on this branch, creating new branch in build repository to push to"
git checkout --orphan ${CI_COMMIT_REF_NAME}-build
ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' | xargs -r git rm --cached
ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' | xargs -r rm -rf
else
echo "Checking out remote branch from last build"
git checkout -b ${REMOTE_BRANCH}-build origin/${REMOTE_BRANCH}-build
fi
cd ../..

View file

@ -0,0 +1,7 @@
from deploy.deploy_message import get_deploy_message
import git
if __name__ == "__main__":
repo = git.Repo(search_parent_directories=True)
msg = get_deploy_message(repo)
print(msg)

71
.gitlab-ci.yml Normal file
View file

@ -0,0 +1,71 @@
stages: # List of stages for jobs, and their order of execution
- get
- build
- pages
get-build-repo: # This job runs in the build stage, which runs first.
stage: get
before_script:
- apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$GITLAB_DEPLOY_KEY"
- echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- git config --global user.email "git@maximilian-kessler.de"
- git config --global user.name "Maximilian Keßler (via gitlab runner)"
script:
- echo "Getting old Build repo..."
- .ci/get_build_repo_from_origin.sh
tags:
- latex
artifacts:
paths:
- build/*
build-packages: # This job runs in the deploy stage.
stage: build # It only runs when *both* jobs in the test stage complete successfully.
before_script:
- apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
- eval $(ssh-agent -s)
- echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- git config --global user.email "git@maximilian-kessler.de"
- git config --global user.name "Maximilian Keßler (via gitlab runner)"
script:
- echo "Building packages incrementally..."
- sh .ci/ci_build.sh
variables:
GIT_SUBMODULE_STRATEGY: recursive
artifacts:
paths:
- build/*
tags:
- latex
pages:
stage: pages
artifacts:
paths:
- public/
script:
- mv build/ public/
tags:
- latex
only:
- master

4
.gitmodules vendored
View file

@ -1,4 +1,4 @@
[submodule "PyTeX"] [submodule "PyTeX"]
path = PyTeX path = PyTeX
url = https://github.com/kesslermaximilian/PyTeX.git url = ../PyTeX.git
branch= latex-packages branch = latex-packages

View file

@ -15,15 +15,8 @@ install:
- pip install GitPython - pip install GitPython
script: script:
- export COMMIT_MSG=$(python3 -c 'from git_version import get_deploy_message; print(get_deploy_message())') - source ./.travis/get_build_repo_from_origin.sh
- python3 -c 'from build import build; build("build/LatexPackages/")' - source ./.travis/travis_build.sh
- cd build
- zip -r LatexPackages.zip LatexPackages
- tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html
- cd ..
after_success:
deploy: deploy:
- provider: pages - provider: pages
@ -42,7 +35,7 @@ deploy:
- provider: pages - provider: pages
edge: true edge: true
github-token: $GITHUB_TOKEN github-token: $GITHUB_TOKEN
local-dir: build/LatexPackages local-dir: build/LatexPackagesBuild
repo: kesslermaximilian/LatexPackagesBuild repo: kesslermaximilian/LatexPackagesBuild
commiter_from_gh: true commiter_from_gh: true
allow_empty_commit: true allow_empty_commit: true

View file

@ -1,7 +1,12 @@
.PHONY: build .PHONY: build
BUILD_FLAGS=--recursive --git-version --pytex-version --license --author "Maximilian Keßler" --pytex-info-text --extra-header ".build/header_info.txt" --name "prepend-author"
BUILD_DIRS= --source-dir src --build-dir build
build: .initsubmodulelock build: .initsubmodulelock
@python3 build.py @python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS}
dirty: .initsubmodulelock
@python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS} --allow-dirty
init: .initsubmodulelock .gitconfiglock init: .initsubmodulelock .gitconfiglock
@ -21,4 +26,7 @@ config: .gitconfiglock
clean: clean:
-rm -r build/ @-rm -r build/
ci-build:
@python3 build.py --source-dir src --build-dir build/LatexPackagesBuild ${BUILD_FLAGS}

2
PyTeX

@ -1 +1 @@
Subproject commit 05f53a917113648a81ab96f353800d64dd9e6ec9 Subproject commit 126d420b7a6ed2d4b2a42d534ab2e12344869643

14
build.py Normal file → Executable file
View file

@ -1,15 +1,7 @@
#! /usr/bin/python3
import sys import sys
from pathlib import Path
from build_scripts.build import build from PyTeX.build import parse_and_build
if __name__ == "__main__": if __name__ == "__main__":
check_existence = True parse_and_build(sys.argv[1:])
if len(sys.argv) == 2:
if sys.argv[1] == '--only-new':
check_existence = False
build(
src_dir=Path('./src').resolve(),
build_dir=Path('./build').resolve(),
check_existence=check_existence
)

View file

@ -1,5 +0,0 @@
from .build import build
__all__ = [
'build'
]

View file

@ -1,67 +0,0 @@
from pathlib import Path
import git
import PyTeX
from build_scripts.git_hook import get_latest_commit, is_recent
from .build_information import build_information
def build(src_dir: Path, build_dir: Path, check_existence: bool = True):
print('[PyTeX] Getting git repository information...')
extra_header, repo_description = build_information()
print('[PyTeX] Building version {version} of LatexPackages'.format(version=repo_description))
print('[PyTeX] Latest commit message: ' + get_latest_commit(git.Repo()).message.strip())
if git.Repo().is_dirty(untracked_files=True):
extra_header += ['WARNING: Local changes to git repository detected.',
' The build will not be reproducible (!)']
num_packages = 0
num_skipped_packages = 0
num_classes = 0
num_skipped_classes = 0
for file in src_dir.rglob('*.pysty'):
output_dir = build_dir / str(file.parent.relative_to(src_dir))
if not is_recent(file, git.Repo()):
if not check_existence:
print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name))
num_skipped_packages += 1
continue
else:
if (output_dir / ('mkessler-' + str(file.with_suffix('.sty').name))).exists():
print('[PyTex] Skipping file {file} since it was not modified since '
'last build and already exists'.format(file=file.name))
num_skipped_packages += 1
continue
num_packages += 1
formatter = PyTeX.PackageFormatter(
package_name=file.with_suffix('').name,
extra_header=extra_header)
print('[PyTeX] Writing file {}'.format(formatter.file_name))
formatter.make_default_macros()
formatter.format_file(file, output_dir)
for file in src_dir.rglob('*.pycls'):
output_dir = build_dir / str(file.parent.relative_to(src_dir))
if not is_recent(file, git.Repo()):
if not check_existence:
print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name))
num_skipped_classes += 1
continue
else:
if (output_dir / ('mkessler-' + str(file.with_suffix('.cls').name))).exists():
print('[PyTex] Skipping file {file} since it was not modified since '
'last build and already exists'.format(file=file.name))
num_skipped_classes += 1
continue
output_dir = build_dir / str(file.parent.relative_to(src_dir))
num_classes += 1
formatter = PyTeX.ClassFormatter(
class_name=file.with_suffix('').name,
extra_header=extra_header)
print('[PyTeX] Writing class file {}'.format(formatter.file_name))
formatter.make_default_macros()
formatter.format_file(input_path=file, output_dir=output_dir)
print(f'[PyTeX] Successfully built {num_packages} packages (skipped {num_skipped_packages}) '
f'and {num_classes} classes (skipped {num_skipped_classes}) in {build_dir}/')

View file

@ -1,20 +0,0 @@
import git
import datetime
from build_scripts.git_hook import git_describe, get_latest_commit
from .config import BUILD_DETAILS
def build_information():
repo = git.Repo()
repo_description = git_describe(get_latest_commit(repo))
pytex_repo = repo.submodule('PyTeX').module()
pytex_repo_description = git_describe(get_latest_commit(pytex_repo))
return list(map(lambda line: line.format(
build_time=datetime.datetime.now().strftime('%Y/%m/%d %H:%M'),
pytex_version=pytex_repo_description,
pytex_commit_hash=get_latest_commit(pytex_repo).hexsha[0:7],
packages_version=repo_description,
packages_commit_hash=get_latest_commit(repo).hexsha[0:7]
), BUILD_DETAILS)), repo_description

View file

@ -1,6 +0,0 @@
BUILD_DETAILS = [
"Build details:",
" Build time: {build_time}",
" PyTeX version: {pytex_version} (commit {pytex_commit_hash})",
" LatexPackages version: {packages_version} (commit {packages_commit_hash})"
]

View file

@ -1,9 +0,0 @@
from .git_version import git_describe, get_history, get_latest_commit
from .recent import is_recent
__all__ = [
'git_describe',
'get_history',
'get_latest_commit',
'is_recent'
]

View file

@ -1,53 +0,0 @@
import git
from typing import Dict
def get_latest_commit(repo):
if repo.head.is_detached:
return repo.head.commit
else:
return repo.head.ref.commit
def get_history(commit: git.objects.commit.Commit, priority=0, depth=0) -> Dict:
commit_history = {commit.hexsha: {
'priority': priority,
'depth': depth
}}
try:
if len(commit.parents) > 0:
commit_history.update(get_history(commit.parents[0], priority, depth + 1))
for parent in commit.parents[1:]:
commit_history.update(get_history(parent, priority + 1, depth + 1))
except ValueError:
pass
return commit_history
def git_describe(commit: git.objects.commit.Commit):
commit_history = get_history(commit)
latest_tag = None
for tag in commit.repo.tags:
sha = tag.commit.hexsha
if sha in commit_history.keys():
if latest_tag is None:
latest_tag = tag
elif commit_history[sha]['priority'] < commit_history[latest_tag.commit.hexsha]['priority']:
latest_tag = tag
elif commit_history[sha]['priority'] > commit_history[latest_tag.commit.hexsha]['priority']:
pass # move on
elif commit_history[sha]['depth'] < commit_history[latest_tag.commit.hexsha]['depth']:
latest_tag = tag
elif commit_history[sha]['depth'] > commit_history[latest_tag.commit.hexsha]['depth']:
pass # move on
elif tag.object.tagged_date > latest_tag.object.tagged_date:
latest_tag = tag
if latest_tag is None:
return "No tags found - cannot describe anything."
else:
msg = latest_tag.name
if commit_history[latest_tag.commit.hexsha]['depth'] != 0:
msg += "-{depth}".format(depth=commit_history[latest_tag.commit.hexsha]['depth'])
if commit.repo.is_dirty(untracked_files=True):
msg += '-*'
return msg

View file

@ -1,14 +0,0 @@
from .git_version import get_latest_commit
def is_recent(file, repo):
modified_files = [item.a_path for item in repo.index.diff(None)]
if file in modified_files:
return True
for parent in get_latest_commit(repo).parents:
newly_committed_files = [item.a_path for item in repo.index.diff(parent)]
if file in newly_committed_files:
return True
return False

View file

@ -1,5 +0,0 @@
from .deploy_message import get_deploy_message
__all__ = [
'get_deploy_message'
]