Merge branch 'build-rework' into 'master'
Build rework See merge request latexci/packages/LatexPackages!4
This commit is contained in:
commit
0e520739a3
20 changed files with 142 additions and 207 deletions
7
.build/header_info.txt
Normal file
7
.build/header_info.txt
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
This LaTeX {latex_file_type} is free software and distributed under the MIT License. You
|
||||||
|
may use it freely for your purposes. The latest version of the {latex_file_type} can be
|
||||||
|
obtained via GitHub under
|
||||||
|
https://github.com/kesslermaximilian/LatexPackages
|
||||||
|
For further information see the url above.
|
||||||
|
Reportings of bugs, suggestions and improvements are welcome, see the README
|
||||||
|
at the Git repository for further information.
|
12
.ci/ci_build.sh
Executable file
12
.ci/ci_build.sh
Executable file
|
@ -0,0 +1,12 @@
|
||||||
|
cd .ci
|
||||||
|
export COMMIT_MSG=$(python3 print_deploy_message.py)
|
||||||
|
cd ..
|
||||||
|
make ci-build
|
||||||
|
cd build
|
||||||
|
zip -r LatexPackages.zip LatexPackagesBuild/ -x '*.git*'
|
||||||
|
tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html
|
||||||
|
cd LatexPackagesBuild
|
||||||
|
git add .
|
||||||
|
git commit -m "${COMMIT_MSG}" || echo "Nothing new to commit"
|
||||||
|
git push --set-upstream origin ${CI_COMMIT_REF_NAME}-build
|
||||||
|
|
|
@ -1,5 +1,11 @@
|
||||||
import os
|
import os
|
||||||
from build_scripts.git_hook import get_latest_commit
|
|
||||||
|
|
||||||
|
def get_latest_commit(repo):
|
||||||
|
if repo.head.is_detached:
|
||||||
|
return repo.head.commit
|
||||||
|
else:
|
||||||
|
return repo.head.ref.commit
|
||||||
|
|
||||||
|
|
||||||
def get_deploy_message(repo):
|
def get_deploy_message(repo):
|
||||||
|
@ -8,6 +14,6 @@ def get_deploy_message(repo):
|
||||||
"\n" \
|
"\n" \
|
||||||
"Build branch {branch} ({hexsha}) from {repo_name}" \
|
"Build branch {branch} ({hexsha}) from {repo_name}" \
|
||||||
.format(old_msg=old_msg,
|
.format(old_msg=old_msg,
|
||||||
branch=os.environ['TRAVIS_BRANCH'],
|
branch=os.environ['CI_COMMIT_REF_NAME'],
|
||||||
hexsha=get_latest_commit(repo).hexsha[0:7],
|
hexsha=get_latest_commit(repo).hexsha[0:7],
|
||||||
repo_name='kesslermaximilian/LatexPackages')
|
repo_name='kesslermaximilian/LatexPackages')
|
18
.ci/get_build_repo_from_origin.sh
Executable file
18
.ci/get_build_repo_from_origin.sh
Executable file
|
@ -0,0 +1,18 @@
|
||||||
|
# ! /bin/sh
|
||||||
|
ssh git@gitlab.com
|
||||||
|
git clone git@gitlab.com:latexci/packages/LatexPackagesBuild.git build/LatexPackagesBuild
|
||||||
|
cd build/LatexPackagesBuild
|
||||||
|
REMOTE_BRANCH=$(git branch -a | sed -n '/remotes\/origin\/.*-build/p' | sed 's/remotes\/origin\///g' | sed 's/-build//g' | sed 's/[[:space:]]//g' | sed -n "/^${CI_COMMIT_REF_NAME}$/p")
|
||||||
|
|
||||||
|
echo ${REMOTE_BRANCH}
|
||||||
|
|
||||||
|
if [ "$REMOTE_BRANCH" = "" ];then
|
||||||
|
echo "This is the first build on this branch, creating new branch in build repository to push to"
|
||||||
|
git checkout --orphan ${CI_COMMIT_REF_NAME}-build
|
||||||
|
ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' | xargs -r git rm --cached
|
||||||
|
ls -ra | sed '/^\.git$/d' | sed '/^\.\.$/d' | sed '/^\.$/d' | xargs -r rm -rf
|
||||||
|
else
|
||||||
|
echo "Checking out remote branch from last build"
|
||||||
|
git checkout -b ${REMOTE_BRANCH}-build origin/${REMOTE_BRANCH}-build
|
||||||
|
fi
|
||||||
|
cd ../..
|
7
.ci/print_deploy_message.py
Normal file
7
.ci/print_deploy_message.py
Normal file
|
@ -0,0 +1,7 @@
|
||||||
|
from deploy.deploy_message import get_deploy_message
|
||||||
|
import git
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
repo = git.Repo(search_parent_directories=True)
|
||||||
|
msg = get_deploy_message(repo)
|
||||||
|
print(msg)
|
71
.gitlab-ci.yml
Normal file
71
.gitlab-ci.yml
Normal file
|
@ -0,0 +1,71 @@
|
||||||
|
stages: # List of stages for jobs, and their order of execution
|
||||||
|
- get
|
||||||
|
- build
|
||||||
|
- pages
|
||||||
|
|
||||||
|
|
||||||
|
get-build-repo: # This job runs in the build stage, which runs first.
|
||||||
|
stage: get
|
||||||
|
before_script:
|
||||||
|
- apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing
|
||||||
|
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
|
||||||
|
- eval $(ssh-agent -s)
|
||||||
|
- echo "$GITLAB_DEPLOY_KEY"
|
||||||
|
- echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
|
||||||
|
|
||||||
|
- mkdir -p ~/.ssh
|
||||||
|
- chmod 700 ~/.ssh
|
||||||
|
|
||||||
|
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
|
||||||
|
- chmod 644 ~/.ssh/known_hosts
|
||||||
|
|
||||||
|
- git config --global user.email "git@maximilian-kessler.de"
|
||||||
|
- git config --global user.name "Maximilian Keßler (via gitlab runner)"
|
||||||
|
script:
|
||||||
|
- echo "Getting old Build repo..."
|
||||||
|
- .ci/get_build_repo_from_origin.sh
|
||||||
|
tags:
|
||||||
|
- latex
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- build/*
|
||||||
|
|
||||||
|
build-packages: # This job runs in the deploy stage.
|
||||||
|
stage: build # It only runs when *both* jobs in the test stage complete successfully.
|
||||||
|
before_script:
|
||||||
|
- apt-get update -y && apt-get install -yqqf openssh-client git unzip sshpass rsync --fix-missing
|
||||||
|
- 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )'
|
||||||
|
- eval $(ssh-agent -s)
|
||||||
|
- echo "$GITLAB_DEPLOY_KEY" | base64 -d | tr -d '\r' | ssh-add - > /dev/null
|
||||||
|
|
||||||
|
- mkdir -p ~/.ssh
|
||||||
|
- chmod 700 ~/.ssh
|
||||||
|
|
||||||
|
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
|
||||||
|
- chmod 644 ~/.ssh/known_hosts
|
||||||
|
|
||||||
|
- git config --global user.email "git@maximilian-kessler.de"
|
||||||
|
- git config --global user.name "Maximilian Keßler (via gitlab runner)"
|
||||||
|
script:
|
||||||
|
- echo "Building packages incrementally..."
|
||||||
|
- sh .ci/ci_build.sh
|
||||||
|
variables:
|
||||||
|
GIT_SUBMODULE_STRATEGY: recursive
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- build/*
|
||||||
|
tags:
|
||||||
|
- latex
|
||||||
|
|
||||||
|
|
||||||
|
pages:
|
||||||
|
stage: pages
|
||||||
|
artifacts:
|
||||||
|
paths:
|
||||||
|
- public/
|
||||||
|
script:
|
||||||
|
- mv build/ public/
|
||||||
|
tags:
|
||||||
|
- latex
|
||||||
|
only:
|
||||||
|
- master
|
4
.gitmodules
vendored
4
.gitmodules
vendored
|
@ -1,4 +1,4 @@
|
||||||
[submodule "PyTeX"]
|
[submodule "PyTeX"]
|
||||||
path = PyTeX
|
path = PyTeX
|
||||||
url = https://github.com/kesslermaximilian/PyTeX.git
|
url = ../PyTeX.git
|
||||||
branch= latex-packages
|
branch = latex-packages
|
||||||
|
|
13
.travis.yml
13
.travis.yml
|
@ -15,15 +15,8 @@ install:
|
||||||
- pip install GitPython
|
- pip install GitPython
|
||||||
|
|
||||||
script:
|
script:
|
||||||
- export COMMIT_MSG=$(python3 -c 'from git_version import get_deploy_message; print(get_deploy_message())')
|
- source ./.travis/get_build_repo_from_origin.sh
|
||||||
- python3 -c 'from build import build; build("build/LatexPackages/")'
|
- source ./.travis/travis_build.sh
|
||||||
- cd build
|
|
||||||
- zip -r LatexPackages.zip LatexPackages
|
|
||||||
- tree -H '.' -I "index.html" -D --charset utf-8 -T "LatexPackages" > index.html
|
|
||||||
- cd ..
|
|
||||||
|
|
||||||
after_success:
|
|
||||||
|
|
||||||
|
|
||||||
deploy:
|
deploy:
|
||||||
- provider: pages
|
- provider: pages
|
||||||
|
@ -42,7 +35,7 @@ deploy:
|
||||||
- provider: pages
|
- provider: pages
|
||||||
edge: true
|
edge: true
|
||||||
github-token: $GITHUB_TOKEN
|
github-token: $GITHUB_TOKEN
|
||||||
local-dir: build/LatexPackages
|
local-dir: build/LatexPackagesBuild
|
||||||
repo: kesslermaximilian/LatexPackagesBuild
|
repo: kesslermaximilian/LatexPackagesBuild
|
||||||
commiter_from_gh: true
|
commiter_from_gh: true
|
||||||
allow_empty_commit: true
|
allow_empty_commit: true
|
||||||
|
|
12
Makefile
12
Makefile
|
@ -1,7 +1,12 @@
|
||||||
.PHONY: build
|
.PHONY: build
|
||||||
|
BUILD_FLAGS=--recursive --git-version --pytex-version --license --author "Maximilian Keßler" --pytex-info-text --extra-header ".build/header_info.txt" --name "prepend-author"
|
||||||
|
BUILD_DIRS= --source-dir src --build-dir build
|
||||||
|
|
||||||
build: .initsubmodulelock
|
build: .initsubmodulelock
|
||||||
@python3 build.py
|
@python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS}
|
||||||
|
|
||||||
|
dirty: .initsubmodulelock
|
||||||
|
@python3 build.py ${BUILD_DIRS} ${BUILD_FLAGS} --allow-dirty
|
||||||
|
|
||||||
init: .initsubmodulelock .gitconfiglock
|
init: .initsubmodulelock .gitconfiglock
|
||||||
|
|
||||||
|
@ -21,4 +26,7 @@ config: .gitconfiglock
|
||||||
|
|
||||||
|
|
||||||
clean:
|
clean:
|
||||||
-rm -r build/
|
@-rm -r build/
|
||||||
|
|
||||||
|
ci-build:
|
||||||
|
@python3 build.py --source-dir src --build-dir build/LatexPackagesBuild ${BUILD_FLAGS}
|
||||||
|
|
2
PyTeX
2
PyTeX
|
@ -1 +1 @@
|
||||||
Subproject commit 05f53a917113648a81ab96f353800d64dd9e6ec9
|
Subproject commit 126d420b7a6ed2d4b2a42d534ab2e12344869643
|
14
build.py
Normal file → Executable file
14
build.py
Normal file → Executable file
|
@ -1,15 +1,7 @@
|
||||||
|
#! /usr/bin/python3
|
||||||
import sys
|
import sys
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
from build_scripts.build import build
|
from PyTeX.build import parse_and_build
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
check_existence = True
|
parse_and_build(sys.argv[1:])
|
||||||
if len(sys.argv) == 2:
|
|
||||||
if sys.argv[1] == '--only-new':
|
|
||||||
check_existence = False
|
|
||||||
build(
|
|
||||||
src_dir=Path('./src').resolve(),
|
|
||||||
build_dir=Path('./build').resolve(),
|
|
||||||
check_existence=check_existence
|
|
||||||
)
|
|
||||||
|
|
|
@ -1,5 +0,0 @@
|
||||||
from .build import build
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
'build'
|
|
||||||
]
|
|
|
@ -1,67 +0,0 @@
|
||||||
from pathlib import Path
|
|
||||||
import git
|
|
||||||
|
|
||||||
import PyTeX
|
|
||||||
|
|
||||||
from build_scripts.git_hook import get_latest_commit, is_recent
|
|
||||||
|
|
||||||
from .build_information import build_information
|
|
||||||
|
|
||||||
|
|
||||||
def build(src_dir: Path, build_dir: Path, check_existence: bool = True):
|
|
||||||
print('[PyTeX] Getting git repository information...')
|
|
||||||
extra_header, repo_description = build_information()
|
|
||||||
print('[PyTeX] Building version {version} of LatexPackages'.format(version=repo_description))
|
|
||||||
print('[PyTeX] Latest commit message: ' + get_latest_commit(git.Repo()).message.strip())
|
|
||||||
if git.Repo().is_dirty(untracked_files=True):
|
|
||||||
extra_header += ['WARNING: Local changes to git repository detected.',
|
|
||||||
' The build will not be reproducible (!)']
|
|
||||||
num_packages = 0
|
|
||||||
num_skipped_packages = 0
|
|
||||||
num_classes = 0
|
|
||||||
num_skipped_classes = 0
|
|
||||||
|
|
||||||
for file in src_dir.rglob('*.pysty'):
|
|
||||||
output_dir = build_dir / str(file.parent.relative_to(src_dir))
|
|
||||||
if not is_recent(file, git.Repo()):
|
|
||||||
if not check_existence:
|
|
||||||
print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name))
|
|
||||||
num_skipped_packages += 1
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
if (output_dir / ('mkessler-' + str(file.with_suffix('.sty').name))).exists():
|
|
||||||
print('[PyTex] Skipping file {file} since it was not modified since '
|
|
||||||
'last build and already exists'.format(file=file.name))
|
|
||||||
num_skipped_packages += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
num_packages += 1
|
|
||||||
formatter = PyTeX.PackageFormatter(
|
|
||||||
package_name=file.with_suffix('').name,
|
|
||||||
extra_header=extra_header)
|
|
||||||
print('[PyTeX] Writing file {}'.format(formatter.file_name))
|
|
||||||
formatter.make_default_macros()
|
|
||||||
formatter.format_file(file, output_dir)
|
|
||||||
for file in src_dir.rglob('*.pycls'):
|
|
||||||
output_dir = build_dir / str(file.parent.relative_to(src_dir))
|
|
||||||
if not is_recent(file, git.Repo()):
|
|
||||||
if not check_existence:
|
|
||||||
print('[PyTex] Skipping file {file} since it was not modified since last build'.format(file=file.name))
|
|
||||||
num_skipped_classes += 1
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
if (output_dir / ('mkessler-' + str(file.with_suffix('.cls').name))).exists():
|
|
||||||
print('[PyTex] Skipping file {file} since it was not modified since '
|
|
||||||
'last build and already exists'.format(file=file.name))
|
|
||||||
num_skipped_classes += 1
|
|
||||||
continue
|
|
||||||
output_dir = build_dir / str(file.parent.relative_to(src_dir))
|
|
||||||
num_classes += 1
|
|
||||||
formatter = PyTeX.ClassFormatter(
|
|
||||||
class_name=file.with_suffix('').name,
|
|
||||||
extra_header=extra_header)
|
|
||||||
print('[PyTeX] Writing class file {}'.format(formatter.file_name))
|
|
||||||
formatter.make_default_macros()
|
|
||||||
formatter.format_file(input_path=file, output_dir=output_dir)
|
|
||||||
print(f'[PyTeX] Successfully built {num_packages} packages (skipped {num_skipped_packages}) '
|
|
||||||
f'and {num_classes} classes (skipped {num_skipped_classes}) in {build_dir}/')
|
|
|
@ -1,20 +0,0 @@
|
||||||
import git
|
|
||||||
import datetime
|
|
||||||
|
|
||||||
from build_scripts.git_hook import git_describe, get_latest_commit
|
|
||||||
|
|
||||||
from .config import BUILD_DETAILS
|
|
||||||
|
|
||||||
|
|
||||||
def build_information():
|
|
||||||
repo = git.Repo()
|
|
||||||
repo_description = git_describe(get_latest_commit(repo))
|
|
||||||
pytex_repo = repo.submodule('PyTeX').module()
|
|
||||||
pytex_repo_description = git_describe(get_latest_commit(pytex_repo))
|
|
||||||
return list(map(lambda line: line.format(
|
|
||||||
build_time=datetime.datetime.now().strftime('%Y/%m/%d %H:%M'),
|
|
||||||
pytex_version=pytex_repo_description,
|
|
||||||
pytex_commit_hash=get_latest_commit(pytex_repo).hexsha[0:7],
|
|
||||||
packages_version=repo_description,
|
|
||||||
packages_commit_hash=get_latest_commit(repo).hexsha[0:7]
|
|
||||||
), BUILD_DETAILS)), repo_description
|
|
|
@ -1,6 +0,0 @@
|
||||||
BUILD_DETAILS = [
|
|
||||||
"Build details:",
|
|
||||||
" Build time: {build_time}",
|
|
||||||
" PyTeX version: {pytex_version} (commit {pytex_commit_hash})",
|
|
||||||
" LatexPackages version: {packages_version} (commit {packages_commit_hash})"
|
|
||||||
]
|
|
|
@ -1,9 +0,0 @@
|
||||||
from .git_version import git_describe, get_history, get_latest_commit
|
|
||||||
from .recent import is_recent
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
'git_describe',
|
|
||||||
'get_history',
|
|
||||||
'get_latest_commit',
|
|
||||||
'is_recent'
|
|
||||||
]
|
|
|
@ -1,53 +0,0 @@
|
||||||
import git
|
|
||||||
from typing import Dict
|
|
||||||
|
|
||||||
|
|
||||||
def get_latest_commit(repo):
|
|
||||||
if repo.head.is_detached:
|
|
||||||
return repo.head.commit
|
|
||||||
else:
|
|
||||||
return repo.head.ref.commit
|
|
||||||
|
|
||||||
|
|
||||||
def get_history(commit: git.objects.commit.Commit, priority=0, depth=0) -> Dict:
|
|
||||||
commit_history = {commit.hexsha: {
|
|
||||||
'priority': priority,
|
|
||||||
'depth': depth
|
|
||||||
}}
|
|
||||||
try:
|
|
||||||
if len(commit.parents) > 0:
|
|
||||||
commit_history.update(get_history(commit.parents[0], priority, depth + 1))
|
|
||||||
for parent in commit.parents[1:]:
|
|
||||||
commit_history.update(get_history(parent, priority + 1, depth + 1))
|
|
||||||
except ValueError:
|
|
||||||
pass
|
|
||||||
return commit_history
|
|
||||||
|
|
||||||
|
|
||||||
def git_describe(commit: git.objects.commit.Commit):
|
|
||||||
commit_history = get_history(commit)
|
|
||||||
latest_tag = None
|
|
||||||
for tag in commit.repo.tags:
|
|
||||||
sha = tag.commit.hexsha
|
|
||||||
if sha in commit_history.keys():
|
|
||||||
if latest_tag is None:
|
|
||||||
latest_tag = tag
|
|
||||||
elif commit_history[sha]['priority'] < commit_history[latest_tag.commit.hexsha]['priority']:
|
|
||||||
latest_tag = tag
|
|
||||||
elif commit_history[sha]['priority'] > commit_history[latest_tag.commit.hexsha]['priority']:
|
|
||||||
pass # move on
|
|
||||||
elif commit_history[sha]['depth'] < commit_history[latest_tag.commit.hexsha]['depth']:
|
|
||||||
latest_tag = tag
|
|
||||||
elif commit_history[sha]['depth'] > commit_history[latest_tag.commit.hexsha]['depth']:
|
|
||||||
pass # move on
|
|
||||||
elif tag.object.tagged_date > latest_tag.object.tagged_date:
|
|
||||||
latest_tag = tag
|
|
||||||
if latest_tag is None:
|
|
||||||
return "No tags found - cannot describe anything."
|
|
||||||
else:
|
|
||||||
msg = latest_tag.name
|
|
||||||
if commit_history[latest_tag.commit.hexsha]['depth'] != 0:
|
|
||||||
msg += "-{depth}".format(depth=commit_history[latest_tag.commit.hexsha]['depth'])
|
|
||||||
if commit.repo.is_dirty(untracked_files=True):
|
|
||||||
msg += '-*'
|
|
||||||
return msg
|
|
|
@ -1,14 +0,0 @@
|
||||||
from .git_version import get_latest_commit
|
|
||||||
|
|
||||||
|
|
||||||
def is_recent(file, repo):
|
|
||||||
modified_files = [item.a_path for item in repo.index.diff(None)]
|
|
||||||
if file in modified_files:
|
|
||||||
return True
|
|
||||||
|
|
||||||
for parent in get_latest_commit(repo).parents:
|
|
||||||
newly_committed_files = [item.a_path for item in repo.index.diff(parent)]
|
|
||||||
if file in newly_committed_files:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
|
@ -1,5 +0,0 @@
|
||||||
from .deploy_message import get_deploy_message
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
'get_deploy_message'
|
|
||||||
]
|
|
Loading…
Reference in a new issue