build+ci: support macOS on arm64
This commit also syncs the CI config and scripts with ndn-cxx
Change-Id: I0403bfcf211e41e15fa59bbbee4cbd1966d9c708
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 47d9a6a..3495a8f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -13,22 +13,24 @@
jobs:
linux:
name: ${{ matrix.compiler }} on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
- compiler: [g++-8, g++-9, g++-10, g++-11,
- clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12]
os: [ubuntu-20.04]
+ compiler: [g++-7, g++-8, g++-9, g++-10,
+ clang++-7, clang++-8, clang++-9, clang++-10, clang++-11, clang++-12]
include:
- - compiler: g++-7
- os: ubuntu-18.04
- - compiler: clang++-6.0
- os: ubuntu-18.04
- runs-on: ${{ matrix.os }}
+ - os: ubuntu-22.04
+ compiler: g++-11
+ - os: ubuntu-22.04
+ compiler: g++-12
+ - os: ubuntu-22.04
+ compiler: clang++-13
+ - os: ubuntu-22.04
+ compiler: clang++-14
env:
CXX: ${{ matrix.compiler }}
- NODE_LABELS: Linux Ubuntu
- WAF_JOBS: 2
steps:
- name: Install C++ compiler
run: |
@@ -41,20 +43,15 @@
macos:
name: Xcode ${{ matrix.xcode }} on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
- xcode: ['11.3', '11.7', '12.4']
- os: [macos-10.15]
+ os: [macos-11]
+ xcode: ['12.4', '12.5', '13.2']
include:
- - xcode: '12.5'
- os: macos-11
- - xcode: '13.2'
- os: macos-11
- runs-on: ${{ matrix.os }}
- env:
- NODE_LABELS: OSX
- WAF_JOBS: 3
+ - os: macos-12
+ xcode: '13.4'
steps:
- name: Set up Xcode
uses: maxim-lobanov/setup-xcode@v1
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 3fb1749..d5fcfb0 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -11,23 +11,18 @@
jobs:
build:
+ runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
- os: [macos-11, ubuntu-20.04]
- runs-on: ${{ matrix.os }}
+ os: [macos-12, ubuntu-20.04]
env:
JOB_NAME: Docs
- WAF_JOBS: 3
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install dependencies
run: |
- case ${RUNNER_OS} in
- Linux) export NODE_LABELS="Linux Ubuntu" ;;
- macOS) export NODE_LABELS="OSX" ;;
- esac
find .jenkins.d/ -type f -name '[1-9]*.sh' -exec chmod -x '{}' +
./.jenkins
- name: Build documentation
diff --git a/.jenkins b/.jenkins
index da10d69..ee16e29 100755
--- a/.jenkins
+++ b/.jenkins
@@ -1,12 +1,30 @@
#!/usr/bin/env bash
-set -e
-source .jenkins.d/util.sh
+set -eo pipefail
-if has Linux $NODE_LABELS; then
- export PATH="${HOME}/.local/bin${PATH:+:}${PATH}"
-fi
+case $(uname) in
+ Linux)
+ if [[ -e /etc/os-release ]]; then
+ source /etc/os-release
+ else
+ source /usr/lib/os-release
+ fi
+ export ID VERSION_ID
+ export ID_LIKE="${ID} ${ID_LIKE} linux"
+ export PATH="${HOME}/.local/bin${PATH:+:}${PATH}"
+ ;;
+ Darwin)
+ # Emulate a subset of os-release(5)
+ export ID=macos
+ export VERSION_ID=$(sw_vers -productVersion)
+ if [[ -x /opt/homebrew/bin/brew ]]; then
+ eval "$(/opt/homebrew/bin/brew shellenv)"
+ elif [[ -x /usr/local/bin/brew ]]; then
+ eval "$(/usr/local/bin/brew shellenv)"
+ fi
+ ;;
+esac
+
export CACHE_DIR=${CACHE_DIR:-/tmp}
-export WAF_JOBS=${WAF_JOBS:-1}
[[ $JOB_NAME == *"code-coverage" ]] && export DISABLE_ASAN=yes
for file in .jenkins.d/*; do
diff --git a/.jenkins.d/00-deps.sh b/.jenkins.d/00-deps.sh
index d3b16f9..12b5de3 100755
--- a/.jenkins.d/00-deps.sh
+++ b/.jenkins.d/00-deps.sh
@@ -1,49 +1,44 @@
#!/usr/bin/env bash
-set -ex
+set -eo pipefail
-if has OSX $NODE_LABELS; then
- FORMULAE=(boost openssl pkg-config)
- if [[ $JOB_NAME == *"Docs" ]]; then
+APT_PKGS=(build-essential pkg-config python3-minimal
+ libboost-all-dev libssl-dev libsqlite3-dev)
+FORMULAE=(boost openssl pkg-config)
+PIP_PKGS=()
+case $JOB_NAME in
+ *code-coverage)
+ APT_PKGS+=(lcov python3-pip)
+ PIP_PKGS+=('gcovr~=5.2')
+ ;;
+ *Docs)
+ APT_PKGS+=(doxygen graphviz python3-pip)
FORMULAE+=(doxygen graphviz)
- fi
+ PIP_PKGS+=(sphinx sphinxcontrib-doxylink)
+ ;;
+esac
+set -x
+
+if [[ $ID == macos ]]; then
if [[ -n $GITHUB_ACTIONS ]]; then
- # GitHub Actions runners have a large number of pre-installed
- # Homebrew packages. Don't waste time upgrading all of them.
- brew list --versions "${FORMULAE[@]}" || brew update
- for FORMULA in "${FORMULAE[@]}"; do
- brew list --versions "$FORMULA" || brew install "$FORMULA"
- done
- # Ensure /usr/local/opt/openssl exists
- brew reinstall openssl
- else
- brew update
- brew upgrade
- brew install "${FORMULAE[@]}"
- brew cleanup
+ export HOMEBREW_NO_INSTALL_UPGRADE=1
+ fi
+ brew update
+ brew install --formula "${FORMULAE[@]}"
+
+ if (( ${#PIP_PKGS[@]} )); then
+ pip3 install --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
fi
- if [[ $JOB_NAME == *"Docs" ]]; then
- pip3 install --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink
- fi
-
-elif has Ubuntu $NODE_LABELS; then
+elif [[ $ID_LIKE == *debian* ]]; then
sudo apt-get -qq update
- sudo apt-get -qy install build-essential pkg-config python3-minimal \
- libboost-all-dev libssl-dev libsqlite3-dev
+ sudo apt-get -qy install "${APT_PKGS[@]}"
- case $JOB_NAME in
- *code-coverage)
- sudo apt-get -qy install lcov python3-pip
- pip3 install --user --upgrade --upgrade-strategy=eager 'gcovr~=5.1'
- ;;
- *Docs)
- sudo apt-get -qy install doxygen graphviz python3-pip
- pip3 install --user --upgrade --upgrade-strategy=eager sphinx sphinxcontrib-doxylink
- ;;
- esac
+ if (( ${#PIP_PKGS[@]} )); then
+ pip3 install --user --upgrade --upgrade-strategy=eager "${PIP_PKGS[@]}"
+ fi
-elif has CentOS $NODE_LABELS; then
+elif [[ $ID_LIKE == *fedora* ]]; then
sudo dnf -y install gcc-c++ libasan pkgconf-pkg-config python3 \
boost-devel openssl-devel sqlite-devel
fi
diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh
index 5da6d35..91ebefd 100755
--- a/.jenkins.d/01-ndn-cxx.sh
+++ b/.jenkins.d/01-ndn-cxx.sh
@@ -1,11 +1,11 @@
#!/usr/bin/env bash
-set -ex
+set -exo pipefail
pushd "$CACHE_DIR" >/dev/null
INSTALLED_VERSION=
-if has OSX $NODE_LABELS; then
- BOOST=$(brew ls --versions boost)
+if [[ $ID == macos ]]; then
+ BOOST=$(brew list --formula --versions boost)
OLD_BOOST=$(cat boost.txt || :)
if [[ $OLD_BOOST != $BOOST ]]; then
echo "$BOOST" > boost.txt
@@ -35,16 +35,16 @@
pushd ndn-cxx >/dev/null
-./waf --color=yes configure --disable-static --enable-shared --without-osx-keychain
-./waf --color=yes build -j$WAF_JOBS
-sudo_preserve_env PATH -- ./waf --color=yes install
+./waf --color=yes configure --without-osx-keychain
+./waf --color=yes build
+sudo ./waf --color=yes install
popd >/dev/null
popd >/dev/null
-if has CentOS $NODE_LABELS; then
+if [[ $ID_LIKE == *fedora* ]]; then
sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64
fi
-if has Linux $NODE_LABELS; then
+if [[ $ID_LIKE == *linux* ]]; then
sudo ldconfig
fi
diff --git a/.jenkins.d/09-cleanup.sh b/.jenkins.d/09-cleanup.sh
index 0b9e5f8..1c308bb 100755
--- a/.jenkins.d/09-cleanup.sh
+++ b/.jenkins.d/09-cleanup.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-set -ex
+set -exo pipefail
PROJ=PSync
diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh
index 23c2987..470c1ba 100755
--- a/.jenkins.d/10-build.sh
+++ b/.jenkins.d/10-build.sh
@@ -1,8 +1,5 @@
#!/usr/bin/env bash
-set -ex
-
-git submodule sync
-git submodule update --init
+set -eo pipefail
if [[ -z $DISABLE_ASAN ]]; then
ASAN="--with-sanitizer=address"
@@ -10,35 +7,40 @@
if [[ $JOB_NAME == *"code-coverage" ]]; then
COVERAGE="--with-coverage"
fi
+if [[ $ID == debian && ${VERSION_ID%%.*} -eq 11 ]]; then
+ LZMA="--without-lzma"
+fi
+
+set -x
if [[ $JOB_NAME != *"code-coverage" && $JOB_NAME != *"limited-build" ]]; then
# Build in release mode with tests
./waf --color=yes configure --with-tests
- ./waf --color=yes build -j$WAF_JOBS
+ ./waf --color=yes build
# Cleanup
./waf --color=yes distclean
# Build in release mode without tests
./waf --color=yes configure
- ./waf --color=yes build -j$WAF_JOBS
+ ./waf --color=yes build
# Cleanup
./waf --color=yes distclean
fi
# Build in debug mode with tests and examples
-./waf --color=yes configure --debug --with-tests --with-examples $ASAN $COVERAGE
-./waf --color=yes build -j$WAF_JOBS
+./waf --color=yes configure --debug --with-tests --with-examples $ASAN $COVERAGE $LZMA
+./waf --color=yes build
# (tests will be run against the debug version)
# Install
-sudo_preserve_env PATH -- ./waf --color=yes install
+sudo ./waf --color=yes install
-if has CentOS $NODE_LABELS; then
+if [[ $ID_LIKE == *fedora* ]]; then
sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64
fi
-if has Linux $NODE_LABELS; then
+if [[ $ID_LIKE == *linux* ]]; then
sudo ldconfig
fi
diff --git a/.jenkins.d/20-tests.sh b/.jenkins.d/20-tests.sh
index 81c00b1..ab14284 100755
--- a/.jenkins.d/20-tests.sh
+++ b/.jenkins.d/20-tests.sh
@@ -1,8 +1,5 @@
#!/usr/bin/env bash
-set -ex
-
-# Prepare environment
-rm -rf ~/.ndn
+set -eo pipefail
# https://github.com/google/sanitizers/wiki/AddressSanitizerFlags
ASAN_OPTIONS="color=always"
@@ -14,10 +11,16 @@
ASAN_OPTIONS+=":strip_path_prefix=${PWD}/"
export ASAN_OPTIONS
+# https://www.boost.org/doc/libs/release/libs/test/doc/html/boost_test/runtime_config/summary.html
export BOOST_TEST_BUILD_INFO=1
export BOOST_TEST_COLOR_OUTPUT=1
export BOOST_TEST_DETECT_MEMORY_LEAK=0
export BOOST_TEST_LOGGER=HRF,test_suite,stdout:XML,all,build/xunit-log.xml
+set -x
+
+# Prepare environment
+rm -rf ~/.ndn
+
# Run unit tests
./build/unit-tests
diff --git a/.jenkins.d/30-coverage.sh b/.jenkins.d/30-coverage.sh
index 38315fa..236d441 100755
--- a/.jenkins.d/30-coverage.sh
+++ b/.jenkins.d/30-coverage.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-set -ex
+set -exo pipefail
if [[ $JOB_NAME == *"code-coverage" ]]; then
# Generate an XML report (Cobertura format) and a detailed HTML report using gcovr
diff --git a/.jenkins.d/40-headers-check.sh b/.jenkins.d/40-headers-check.sh
index ad23cee..7deb991 100755
--- a/.jenkins.d/40-headers-check.sh
+++ b/.jenkins.d/40-headers-check.sh
@@ -1,9 +1,7 @@
#!/usr/bin/env bash
-
+set -eo pipefail
# It's intentional not to use `set -x`, because this script explicitly prints useful information
# and should not run in trace mode.
-# It's intentional not to use `set -e`, because this script wants to check all headers
-# (similar to running all test cases), instead of failing at the first error.
PROJ=PSync
PCFILE=PSync
@@ -13,7 +11,7 @@
exit 0
fi
-if has CentOS $NODE_LABELS; then
+if [[ $ID_LIKE == *fedora* ]]; then
export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig
fi
@@ -28,9 +26,8 @@
NERRORS=0
while IFS= read -r -d '' H; do
echo "Checking header ${H#${INCLUDEDIR}/}"
- "$CXX" -xc++ $STD $CXXFLAGS -c -o /dev/null "$H"
- [[ $? -eq 0 ]] || ((NERRORS++))
- ((NCHECKED++))
+ "$CXX" -xc++ $STD $CXXFLAGS -c -o /dev/null "$H" || : $((NERRORS++))
+ : $((NCHECKED++))
done < <(find "$INCLUDEDIR" -name '*.hpp' -type f -print0 2>/dev/null)
if [[ $NCHECKED -eq 0 ]]; then
diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md
index e8dbf37..385af34 100644
--- a/.jenkins.d/README.md
+++ b/.jenkins.d/README.md
@@ -1,28 +1,36 @@
-# CONTINUOUS INTEGRATION SCRIPTS
+# Continuous Integration Scripts
-## Environment Variables Used in Build Scripts
+## Environment Variables
-- `NODE_LABELS`: space-separated list of platform properties. The included values are used by
- the build scripts to select the proper behavior for different operating systems and versions.
+- `ID`: lower-case string that identifies the operating system, for example: `ID=ubuntu`,
+ `ID=centos`. See [os-release(5)] for more information. On macOS, where `os-release` is
+ not available, we emulate it by setting `ID=macos`.
- The list should normally contain `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`.
+- `ID_LIKE`: space-separated list of operating system identifiers that are closely related
+ to the running OS. See [os-release(5)] for more information. The listed values are used
+ by the CI scripts to select the proper behavior for different platforms and OS flavors.
- Example values:
+ Examples:
- - `[OS_TYPE]`: `Linux`, `OSX`
- - `[DISTRO_TYPE]`: `Ubuntu`, `CentOS`
- - `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`, `CentOS-8`, `OSX-10.14`, `OSX-10.15`
+ - On CentOS, `ID_LIKE="centos rhel fedora linux"`
+ - On Ubuntu, `ID_LIKE="ubuntu debian linux"`
-- `JOB_NAME`: optional variable that defines the type of build job. Depending on the job type,
- the build scripts can perform different tasks.
+- `VERSION_ID`: identifies the operating system version, excluding any release code names.
+ See [os-release(5)] for more information. Examples: `VERSION_ID=42`, `VERSION_ID=22.04`.
- Possible values:
+- `JOB_NAME`: defines the type of the current CI job. Depending on the job type, the CI
+ scripts can perform different tasks.
+
+ Supported values:
- empty: default build task
- - `code-coverage`: debug build with tests and code coverage analysis (Ubuntu Linux is assumed)
+ - `code-coverage`: debug build with tests and code coverage analysis
- `limited-build`: only a single debug build with tests
-- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled version
- of ndn-cxx. If not set, `/tmp` is used.
+- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled
+ version of ndn-cxx. If not set, `/tmp` is used.
-- `WAF_JOBS`: number of parallel build threads used by waf, defaults to 1.
+- `DISABLE_ASAN`: disable building with AddressSanitizer. This is automatically set for
+ the `code-coverage` job type.
+
+[os-release(5)]: https://www.freedesktop.org/software/systemd/man/os-release.html
diff --git a/.jenkins.d/util.sh b/.jenkins.d/util.sh
deleted file mode 100644
index 8077a74..0000000
--- a/.jenkins.d/util.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-has() {
- local saved_xtrace
- [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x
- set +x
-
- local p=$1
- shift
- local i ret=1
- for i in "$@"; do
- if [[ "${i}" == "${p}" ]]; then
- ret=0
- break
- fi
- done
-
- set ${saved_xtrace}
- return ${ret}
-}
-export -f has
-
-sudo_preserve_env() {
- local saved_xtrace
- [[ $- == *x* ]] && saved_xtrace=-x || saved_xtrace=+x
- set +x
-
- local vars=()
- while [[ $# -gt 0 ]]; do
- local arg=$1
- shift
- case ${arg} in
- --) break ;;
- *) vars+=("${arg}=${!arg}") ;;
- esac
- done
-
- set ${saved_xtrace}
- sudo env "${vars[@]}" "$@"
-}
-export -f sudo_preserve_env
diff --git a/.waf-tools/boost.py b/.waf-tools/boost.py
index 4b2ede5..a6cdabe 100644
--- a/.waf-tools/boost.py
+++ b/.waf-tools/boost.py
@@ -54,8 +54,8 @@
from waflib.Configure import conf
from waflib.TaskGen import feature, after_method
-BOOST_LIBS = ['/usr/lib', '/usr/local/lib', '/opt/local/lib', '/sw/lib', '/lib']
-BOOST_INCLUDES = ['/usr/include', '/usr/local/include', '/opt/local/include', '/sw/include']
+BOOST_LIBS = ['/usr/lib', '/usr/local/lib', '/opt/homebrew/lib', '/opt/local/lib', '/sw/lib', '/lib']
+BOOST_INCLUDES = ['/usr/include', '/usr/local/include', '/opt/homebrew/include', '/opt/local/include', '/sw/include']
BOOST_VERSION_FILE = 'boost/version.hpp'
BOOST_VERSION_CODE = '''
diff --git a/.waf-tools/default-compiler-flags.py b/.waf-tools/default-compiler-flags.py
index 7c6d282..3ba2dc3 100644
--- a/.waf-tools/default-compiler-flags.py
+++ b/.waf-tools/default-compiler-flags.py
@@ -33,7 +33,7 @@
'The minimum supported clang version is 6.0.')
conf.flags = ClangFlags()
else:
- warnmsg = '%s compiler is unsupported' % cxx
+ warnmsg = f'{cxx} compiler is unsupported'
conf.flags = CompilerFlags()
if errmsg:
@@ -200,7 +200,8 @@
flags = super(ClangFlags, self).getGeneralFlags(conf)
if Utils.unversioned_sys_platform() == 'darwin':
# Bug #4296
- flags['CXXFLAGS'] += [['-isystem', '/usr/local/include'], # for Homebrew
+ brewdir = '/opt/homebrew' if platform.machine() == 'arm64' else '/usr/local'
+ flags['CXXFLAGS'] += [['-isystem', f'{brewdir}/include'], # for Homebrew
['-isystem', '/opt/local/include']] # for MacPorts
elif Utils.unversioned_sys_platform() == 'freebsd':
# Bug #4790