From 0eb6c189a3576aa8745f471c1ad23b3ca123cc75 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jun 2021 15:24:11 +0000 Subject: ci: use the new GitHub Action to download git-sdk-64-minimal In our continuous builds, Windows is the odd cookie that requires a complete development environment to be downloaded because there is no suitable one installed by default on Windows. Side note: technically, there _is_ a development environment present in GitHub Actions' build agents: MSYS2. But it differs from Git for Windows' SDK in subtle points, unfortunately enough so to prevent Git's test suite from running without failures. Traditionally, we support downloading this environment (which we nicknamed `git-sdk-64-minimal`) via a PowerShell scriptlet that accesses the build artifacts of a dedicated Azure Pipeline (which packages a tiny subset of the full Git for Windows SDK, containing just enough to build Git and run its test suite). This PowerShell script is unfortunately not very robust and sometimes fails due to network issues. Of course, we could add code to detect that situation, wait a little, try again, if it fails again wait a little longer, lather, rinse and repeat. Instead of doing all of this in Git's own `.github/workflows/`, though, let's offload this logic to the new GitHub Action at https://github.com/marketplace/actions/setup-git-for-windows-sdk This Action not only downloads and extracts git-sdk-64-minimal _outside_ the worktree (making it no longer necessary to meddle with `.gitignore` or `.git/info/exclude`), it also adds the `bash.exe` to the `PATH` and sets the environment variable `MSYSTEM` (an implementation detail that Git's workflow should never have needed to know about). This allows us to convert all those funny PowerShell tasks that wanted to call git-sdk-64-minimal's `bash.exe`: they all are now regular `bash` scriptlets. This finally lets us get rid of the funny quoting and escaping where we had to pay attention not only to quote and escape the Bash scriptlets properly, but also to add a second level of escaping (with backslashes for double quotes and backticks for dollar signs) to stop PowerShell from doing unintended things. Further, this Action uses a fast caching strategy native to GitHub Actions that should accelerate the download across CI runs: git-sdk-64-minimal is usually updated once per 24h, and needs to be cached only once within that period. Caching it (unfortunately only on a per-branch basis) speeds up the download step, and makes it much more robust at the same time by virtue of accessing a cache location that is closer in the network topology. With this we can drop the home-rolled caching where we try to accelerate the test phase by uploading git-sdk-64-minimal as a workflow artifact after using it to build Git, and then download it as workflow artifact in the test phase. Even better: the `vs-test` job no longer needs to depend on the `windows-build` job. The only reason it depended on it was to ensure that the `git-sdk-64-minimal` workflow artifact was available. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 103 +++++++++------------------------------------ 1 file changed, 19 insertions(+), 84 deletions(-) (limited to '.github/workflows/main.yml') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 73856bafc9..c62766e7b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -82,43 +82,18 @@ jobs: runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - shell: bash - run: | - ## Get artifact - urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r ".value[] | .id") - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')" - curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \ - -o artifacts.zip "$download_url" - - ## Unzip and remove the artifact - unzip artifacts.zip - rm artifacts.zip + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: build - shell: powershell + shell: bash env: HOME: ${{runner.workspace}} - MSYSTEM: MINGW64 NO_PERL: 1 - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude - - ci/make-test-artifacts.sh artifacts - "@ + run: ci/make-test-artifacts.sh artifacts - name: upload build artifacts uses: actions/upload-artifact@v1 with: name: windows-artifacts path: artifacts - - name: upload git-sdk-64-minimal - uses: actions/upload-artifact@v1 - with: - name: git-sdk-64-minimal - path: git-sdk-64-minimal windows-test: runs-on: windows-latest needs: [windows-build] @@ -136,25 +111,14 @@ jobs: - name: extract build artifacts shell: bash run: tar xf artifacts.tar.gz - - name: download git-sdk-64-minimal - uses: actions/download-artifact@v1 - with: - name: git-sdk-64-minimal - path: ${{github.workspace}}/git-sdk-64-minimal/ + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: test - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - # Let Git ignore the SDK - printf '%s\n' /git-sdk-64-minimal/ >>.git/info/exclude - - ci/run-test-slice.sh ${{matrix.nr}} 10 - "@ + shell: bash + run: ci/run-test-slice.sh ${{matrix.nr}} 10 - name: ci/print-test-failures.sh if: failure() - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + shell: bash + run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 @@ -165,27 +129,12 @@ jobs: needs: ci-config if: needs.ci-config.outputs.enabled == 'yes' env: - MSYSTEM: MINGW64 NO_PERL: 1 GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - shell: bash - run: | - ## Get artifact - urlbase=https://dev.azure.com/git-for-windows/git/_apis/build/builds - id=$(curl "$urlbase?definitions=22&statusFilter=completed&resultFilter=succeeded&\$top=1" | - jq -r ".value[] | .id") - download_url="$(curl "$urlbase/$id/artifacts" | - jq -r '.value[] | select(.name == "git-sdk-64-minimal").resource.downloadUrl')" - curl --connect-timeout 10 --retry 5 --retry-delay 0 --retry-max-time 240 \ - -o artifacts.zip "$download_url" - - ## Unzip and remove the artifact - unzip artifacts.zip - rm artifacts.zip + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: initialize vcpkg uses: actions/checkout@v2 with: @@ -211,19 +160,17 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=`pwd`/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar - shell: powershell + shell: bash env: MSVC: 1 VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg run: | - & git-sdk-64-minimal\usr\bin\bash.exe -lc @" - mkdir -p artifacts && - eval \"`$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)\" - "@ + mkdir -p artifacts && + eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - name: upload build artifacts uses: actions/upload-artifact@v1 with: @@ -231,18 +178,14 @@ jobs: path: artifacts vs-test: runs-on: windows-latest - needs: [vs-build, windows-build] + needs: vs-build strategy: fail-fast: false matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - uses: actions/checkout@v1 - - name: download git-sdk-64-minimal - uses: actions/download-artifact@v1 - with: - name: git-sdk-64-minimal - path: ${{github.workspace}}/git-sdk-64-minimal/ + - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: download build artifacts uses: actions/download-artifact@v1 with: @@ -252,23 +195,15 @@ jobs: shell: bash run: tar xf artifacts.tar.gz - name: test - shell: powershell + shell: bash env: - MSYSTEM: MINGW64 NO_SVN_TESTS: 1 GIT_TEST_SKIP_REBASE_P: 1 - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc @" - # Let Git ignore the SDK and the test-cache - printf '%s\n' /git-sdk-64-minimal/ /test-cache/ >>.git/info/exclude - - ci/run-test-slice.sh ${{matrix.nr}} 10 - "@ + run: ci/run-test-slice.sh ${{matrix.nr}} 10 - name: ci/print-test-failures.sh if: failure() - shell: powershell - run: | - & .\git-sdk-64-minimal\usr\bin\bash.exe -lc ci/print-test-failures.sh + shell: bash + run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' uses: actions/upload-artifact@v1 -- cgit v1.2.3 From abb2b389f7604704d3e78ae24e52e1a9429b699d Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jun 2021 15:24:12 +0000 Subject: ci (vs-build): use `cmd` to copy the DLLs, not `powershell` We use a `.bat` script to copy the DLLs in the `vs-build` job, and those type of scripts are native to CMD, not to PowerShell. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to '.github/workflows/main.yml') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c62766e7b1..d430c4e0d2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,10 +152,8 @@ jobs: - name: add msbuild to PATH uses: microsoft/setup-msbuild@v1 - name: copy dlls to root - shell: powershell - run: | - & compat\vcbuild\vcpkg_copy_dlls.bat release - if (!$?) { exit(1) } + shell: cmd + run: compat\vcbuild\vcpkg_copy_dlls.bat release - name: generate Visual Studio solution shell: bash run: | -- cgit v1.2.3 From e9f79acb28cca67e4c8c13148501e612b7f1dd56 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Wed, 23 Jun 2021 15:24:13 +0000 Subject: ci: upgrade to using actions/{up,down}load-artifacts v2 The GitHub Actions to upload/download workflow artifacts saw a major upgrade since Git's GitHub workflow was established. Let's use it. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to '.github/workflows/main.yml') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d430c4e0d2..a399114c0f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,7 +90,7 @@ jobs: NO_PERL: 1 run: ci/make-test-artifacts.sh artifacts - name: upload build artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: windows-artifacts path: artifacts @@ -104,7 +104,7 @@ jobs: steps: - uses: actions/checkout@v1 - name: download build artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: windows-artifacts path: ${{github.workspace}} @@ -121,7 +121,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -170,7 +170,7 @@ jobs: mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - name: upload build artifacts - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: vs-artifacts path: artifacts @@ -185,7 +185,7 @@ jobs: - uses: actions/checkout@v1 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: download build artifacts - uses: actions/download-artifact@v1 + uses: actions/download-artifact@v2 with: name: vs-artifacts path: ${{github.workspace}} @@ -204,7 +204,7 @@ jobs: run: ci/print-test-failures.sh - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-windows path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -242,7 +242,7 @@ jobs: if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} @@ -269,7 +269,7 @@ jobs: if: failure() - name: Upload failed tests' directories if: failure() && env.FAILED_TEST_ARTIFACTS != '' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2 with: name: failed-tests-${{matrix.vector.jobname}} path: ${{env.FAILED_TEST_ARTIFACTS}} -- cgit v1.2.3 From d681d0dc3a77016caa7e26abfe734afbdab44de5 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 4 Jul 2021 22:55:11 +0000 Subject: ci (windows): transfer also the Git-tracked files to the test jobs Git's test suite is excruciatingly slow on Windows, mainly due to the fact that it executes a lot of shell script code, and that's simply not native to Windows. To help with that, we established the pattern where the artifacts are first built in one job, and then multiple test jobs run in parallel using the artifacts built in the first job. We take pains in transferring only the build outputs, and letting `actions/checkout` fill in the rest of the files. One major downside of that strategy is that the test jobs might fail to check out the intended revision (e.g. because the branch has been updated while the build was running, as is frequently the case with the `seen` branch). Let's transfer also the files tracked by Git, and skip the checkout step in the test jobs. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to '.github/workflows/main.yml') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a399114c0f..0f7516c9ef 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,9 @@ jobs: HOME: ${{runner.workspace}} NO_PERL: 1 run: ci/make-test-artifacts.sh artifacts - - name: upload build artifacts + - name: zip up tracked files + run: git archive -o artifacts/tracked.tar.gz HEAD + - name: upload tracked files and build artifacts uses: actions/upload-artifact@v2 with: name: windows-artifacts @@ -102,15 +104,14 @@ jobs: matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - uses: actions/checkout@v1 - - name: download build artifacts + - name: download tracked files and build artifacts uses: actions/download-artifact@v2 with: name: windows-artifacts path: ${{github.workspace}} - - name: extract build artifacts + - name: extract tracked files and build artifacts shell: bash - run: tar xf artifacts.tar.gz + run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: test shell: bash @@ -169,7 +170,9 @@ jobs: run: | mkdir -p artifacts && eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" - - name: upload build artifacts + - name: zip up tracked files + run: git archive -o artifacts/tracked.tar.gz HEAD + - name: upload tracked files and build artifacts uses: actions/upload-artifact@v2 with: name: vs-artifacts @@ -182,16 +185,15 @@ jobs: matrix: nr: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] steps: - - uses: actions/checkout@v1 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - - name: download build artifacts + - name: download tracked files and build artifacts uses: actions/download-artifact@v2 with: name: vs-artifacts path: ${{github.workspace}} - - name: extract build artifacts + - name: extract tracked files and build artifacts shell: bash - run: tar xf artifacts.tar.gz + run: tar xf artifacts.tar.gz && tar xf tracked.tar.gz - name: test shell: bash env: -- cgit v1.2.3 From 9ab0b6612918b833bdec775f7c9bf22c4c4ddd07 Mon Sep 17 00:00:00 2001 From: Dennis Ameling Date: Sun, 4 Jul 2021 22:55:13 +0000 Subject: ci (vs-build): build with NO_GETTEXT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We already build Git for Windows with `NO_GETTEXT` when compiling with GCC. Let's do the same with Visual C, too. Note that we do not technically _need_ to pass `NO_GETTEXT` explicitly in that `make artifacts-tar` invocation because we do this while `MSVC` is set (which will set `uname_S := Windows`, which in turn will set `NO_GETTEXT = YesPlease`). But it is definitely nicer to be explicit here. Signed-off-by: Dennis Ameling Helped-by: Matthias Aßhauer Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to '.github/workflows/main.yml') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f7516c9ef..c99628681e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -159,7 +159,7 @@ jobs: shell: bash run: | cmake `pwd`/contrib/buildsystems/ -DCMAKE_PREFIX_PATH=`pwd`/compat/vcbuild/vcpkg/installed/x64-windows \ - -DMSGFMT_EXE=C:/git-sdk-64-minimal/mingw64/bin/msgfmt.exe -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON + -DNO_GETTEXT=YesPlease -DPERL_TESTS=OFF -DPYTHON_TESTS=OFF -DCURL_NO_CURL_CMAKE=ON - name: MSBuild run: msbuild git.sln -property:Configuration=Release -property:Platform=x64 -maxCpuCount:4 -property:PlatformToolset=v142 - name: bundle artifact tar @@ -169,7 +169,7 @@ jobs: VCPKG_ROOT: ${{github.workspace}}\compat\vcbuild\vcpkg run: | mkdir -p artifacts && - eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts 2>&1 | grep ^tar)" + eval "$(make -n artifacts-tar INCLUDE_DLLS_IN_ARTIFACTS=YesPlease ARTIFACTS_DIRECTORY=artifacts NO_GETTEXT=YesPlease 2>&1 | grep ^tar)" - name: zip up tracked files run: git archive -o artifacts/tracked.tar.gz HEAD - name: upload tracked files and build artifacts -- cgit v1.2.3 From 0dc787a9f23f78902b8792e44ff8187398e49de8 Mon Sep 17 00:00:00 2001 From: Johannes Schindelin Date: Sun, 4 Jul 2021 22:55:14 +0000 Subject: ci: accelerate the checkout By upgrading from v1 to v2 of `actions/checkout`, we avoid fetching all the tags and the complete history: v2 only fetches one revision by default. This should make things a lot faster. Note that `actions/checkout@v2` seems to be incompatible with running in containers: https://github.com/actions/checkout/issues/151. Therefore, we stick with v1 there. Signed-off-by: Johannes Schindelin Signed-off-by: Junio C Hamano --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to '.github/workflows/main.yml') diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c99628681e..e6f99e29a3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -81,7 +81,7 @@ jobs: if: needs.ci-config.outputs.enabled == 'yes' runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: build shell: bash @@ -134,7 +134,7 @@ jobs: GIT_CONFIG_PARAMETERS: "'user.name=CI' 'user.email=ci@git'" runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - uses: git-for-windows/setup-git-for-windows-sdk@v1 - name: initialize vcpkg uses: actions/checkout@v2 @@ -237,7 +237,7 @@ jobs: jobname: ${{matrix.vector.jobname}} runs-on: ${{matrix.vector.pool}} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-build-and-tests.sh - run: ci/print-test-failures.sh @@ -282,7 +282,7 @@ jobs: jobname: StaticAnalysis runs-on: ubuntu-18.04 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/run-static-analysis.sh documentation: @@ -292,6 +292,6 @@ jobs: jobname: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - run: ci/install-dependencies.sh - run: ci/test-documentation.sh -- cgit v1.2.3