build+ci: switch to python3
Also in this commit:
* Sync CI scripts with other projects
* Upgrade waf to version 2.0.19
* Modernize docs/conf.py
* Improve README.md and INSTALL.rst
Refs: #5095
Change-Id: Ic0f4f3be07d7266e57d6565a665eb24a757291e8
diff --git a/.jenkins.d/00-deps.sh b/.jenkins.d/00-deps.sh
index 84c8ae1..458bbd0 100755
--- a/.jenkins.d/00-deps.sh
+++ b/.jenkins.d/00-deps.sh
@@ -1,31 +1,32 @@
#!/usr/bin/env bash
-set -e
-
-JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "$JDIR"/util.sh
-
-set -x
+set -ex
if has OSX $NODE_LABELS; then
FORMULAE=(boost openssl pkg-config)
- brew update
- if [[ -n $TRAVIS ]]; then
- # travis images come with a large number of brew packages
- # pre-installed, don't waste time upgrading all of them
- for FORMULA in "${FORMULAE[@]}"; do
- brew outdated $FORMULA || brew upgrade $FORMULA
- done
- else
- brew upgrade
+ if has OSX-10.13 $NODE_LABELS || has OSX-10.14 $NODE_LABELS; then
+ FORMULAE+=(python)
fi
- brew install "${FORMULAE[@]}"
- brew cleanup
-fi
-if has Ubuntu $NODE_LABELS; then
+ if [[ -n $TRAVIS ]]; then
+ # Travis images come with a large number of pre-installed
+ # brew packages, don't waste time upgrading all of them
+ brew list --versions "${FORMULAE[@]}" || brew update
+ for FORMULA in "${FORMULAE[@]}"; do
+ brew list --versions "$FORMULA" || brew install "$FORMULA"
+ done
+ # Ensure /usr/local/opt/openssl exists
+ brew reinstall openssl
+ else
+ brew update
+ brew upgrade
+ brew install "${FORMULAE[@]}"
+ brew cleanup
+ fi
+
+elif has Ubuntu $NODE_LABELS; then
sudo apt-get -qq update
- sudo apt-get -qy install build-essential pkg-config libboost-all-dev \
- libsqlite3-dev libssl-dev
+ sudo apt-get -qy install g++ pkg-config python3-minimal \
+ libboost-all-dev libssl-dev libsqlite3-dev
if [[ $JOB_NAME == *"code-coverage" ]]; then
sudo apt-get -qy install gcovr lcov libgd-perl
diff --git a/.jenkins.d/01-ndn-cxx.sh b/.jenkins.d/01-ndn-cxx.sh
index 29a41ea..4e0e154 100755
--- a/.jenkins.d/01-ndn-cxx.sh
+++ b/.jenkins.d/01-ndn-cxx.sh
@@ -1,12 +1,7 @@
#!/usr/bin/env bash
-set -e
+set -ex
-JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "$JDIR"/util.sh
-
-set -x
-
-pushd "${CACHE_DIR:-/tmp}" >/dev/null
+pushd "$CACHE_DIR" >/dev/null
INSTALLED_VERSION=
if has OSX $NODE_LABELS; then
@@ -22,35 +17,39 @@
INSTALLED_VERSION=$(git -C ndn-cxx rev-parse HEAD 2>/dev/null || echo NONE)
fi
-sudo rm -Rf ndn-cxx-latest
-
-git clone --depth 1 git://github.com/named-data/ndn-cxx ndn-cxx-latest
-
+sudo rm -rf ndn-cxx-latest
+git clone --depth 1 https://github.com/named-data/ndn-cxx.git ndn-cxx-latest
LATEST_VERSION=$(git -C ndn-cxx-latest rev-parse HEAD 2>/dev/null || echo UNKNOWN)
if [[ $INSTALLED_VERSION != $LATEST_VERSION ]]; then
- sudo rm -Rf ndn-cxx
+ sudo rm -rf ndn-cxx
mv ndn-cxx-latest ndn-cxx
else
- sudo rm -Rf ndn-cxx-latest
+ sudo rm -rf ndn-cxx-latest
fi
sudo rm -f /usr/local/bin/ndnsec*
sudo rm -fr /usr/local/include/ndn-cxx
-sudo rm -f /usr/local/lib/libndn-cxx*
-sudo rm -f /usr/local/lib/pkgconfig/libndn-cxx.pc
+sudo rm -f /usr/local/lib{,64}/libndn-cxx*
+sudo rm -f /usr/local/lib{,64}/pkgconfig/libndn-cxx.pc
pushd ndn-cxx >/dev/null
-./waf configure --color=yes --enable-shared --disable-static --without-osx-keychain
-./waf build --color=yes -j${WAF_JOBS:-1}
-sudo_preserve_env PATH -- ./waf install --color=yes
+if has CentOS-8 $NODE_LABELS; then
+ # https://bugzilla.redhat.com/show_bug.cgi?id=1721553
+ PCH="--without-pch"
+fi
+
+./waf --color=yes configure --disable-static --enable-shared --without-osx-keychain $PCH
+./waf --color=yes build -j$WAF_JOBS
+sudo_preserve_env PATH -- ./waf --color=yes install
popd >/dev/null
popd >/dev/null
+if has CentOS-8 $NODE_LABELS; then
+ sudo tee /etc/ld.so.conf.d/ndn.conf >/dev/null <<< /usr/local/lib64
+fi
if has Linux $NODE_LABELS; then
sudo ldconfig
-elif has FreeBSD10 $NODE_LABELS; then
- sudo ldconfig -m
fi
diff --git a/.jenkins.d/10-build.sh b/.jenkins.d/10-build.sh
index 0c952ec..2a60f0d 100755
--- a/.jenkins.d/10-build.sh
+++ b/.jenkins.d/10-build.sh
@@ -1,41 +1,37 @@
#!/usr/bin/env bash
-set -e
+set -ex
-JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "$JDIR"/util.sh
+git submodule sync
+git submodule update --init
-set -x
-
-if [[ $JOB_NAME == *"code-coverage" ]]; then
- COVERAGE="--with-coverage"
-elif [[ -z $DISABLE_ASAN ]]; then
+if [[ -z $DISABLE_ASAN ]]; then
ASAN="--with-sanitizer=address"
fi
-
-# Cleanup
-sudo_preserve_env PATH -- ./waf --color=yes distclean
-
-if [[ $JOB_NAME != *"code-coverage" && $JOB_NAME != *"limited-build" ]]; then
- # Configure/build in optimized mode with tests
- ./waf --color=yes configure --with-tests
- ./waf --color=yes build -j${WAF_JOBS:-1}
-
- # Cleanup
- sudo_preserve_env PATH -- ./waf --color=yes distclean
-
- # Configure/build in optimized mode without tests
- ./waf --color=yes configure
- ./waf --color=yes build -j${WAF_JOBS:-1}
-
- # Cleanup
- sudo_preserve_env PATH -- ./waf --color=yes distclean
+if [[ $JOB_NAME == *"code-coverage" ]]; then
+ COVERAGE="--with-coverage"
fi
-# Configure/build in debug mode with tests
-./waf --color=yes configure --debug --with-tests $ASAN $COVERAGE
-./waf --color=yes build -j${WAF_JOBS:-1}
+if [[ $JOB_NAME != *"code-coverage" && $JOB_NAME != *"limited-build" ]]; then
+ # Build in release mode with tests
+ ./waf --color=yes configure --with-tests
+ ./waf --color=yes build -j$WAF_JOBS
-# (tests will be run against debug version)
+ # Cleanup
+ ./waf --color=yes distclean
+
+ # Build in release mode without tests
+ ./waf --color=yes configure
+ ./waf --color=yes build -j$WAF_JOBS
+
+ # Cleanup
+ ./waf --color=yes distclean
+fi
+
+# Build in debug mode with tests
+./waf --color=yes configure --debug --with-tests $ASAN $COVERAGE
+./waf --color=yes build -j$WAF_JOBS
+
+# (tests will be run against the debug version)
# Install
sudo_preserve_env PATH -- ./waf --color=yes install
diff --git a/.jenkins.d/20-tests.sh b/.jenkins.d/20-tests.sh
index ec4461b..d4b9652 100755
--- a/.jenkins.d/20-tests.sh
+++ b/.jenkins.d/20-tests.sh
@@ -1,13 +1,8 @@
#!/usr/bin/env bash
-set -e
-
-JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "$JDIR"/util.sh
-
-set -x
+set -ex
# Prepare environment
-rm -Rf ~/.ndn
+rm -rf ~/.ndn
if has OSX $NODE_LABELS; then
security unlock-keychain -p named-data
@@ -15,7 +10,7 @@
ndnsec-keygen "/tmp/jenkins/$NODE_NAME" | ndnsec-install-cert -
-BOOST_VERSION=$(python -c "import sys; sys.path.append('build/c4che'); import _cache; print(_cache.BOOST_VERSION_NUMBER);")
+BOOST_VERSION=$(python3 -c "import sys; sys.path.append('build/c4che'); import _cache; print(_cache.BOOST_VERSION_NUMBER);")
ut_log_args() {
if (( BOOST_VERSION >= 106200 )); then
@@ -30,13 +25,13 @@
fi
}
+# https://github.com/google/sanitizers/wiki/AddressSanitizerFlags
ASAN_OPTIONS="color=always"
-ASAN_OPTIONS+=":detect_stack_use_after_return=true"
-ASAN_OPTIONS+=":check_initialization_order=true"
-ASAN_OPTIONS+=":strict_init_order=true"
-ASAN_OPTIONS+=":detect_invalid_pointer_pairs=1"
-ASAN_OPTIONS+=":detect_container_overflow=false"
-ASAN_OPTIONS+=":strict_string_checks=true"
+ASAN_OPTIONS+=":check_initialization_order=1"
+ASAN_OPTIONS+=":detect_stack_use_after_return=1"
+ASAN_OPTIONS+=":strict_init_order=1"
+ASAN_OPTIONS+=":strict_string_checks=1"
+ASAN_OPTIONS+=":detect_invalid_pointer_pairs=2"
ASAN_OPTIONS+=":strip_path_prefix=${PWD}/"
export ASAN_OPTIONS
diff --git a/.jenkins.d/30-coverage.sh b/.jenkins.d/30-coverage.sh
index 9990760..7f8ae58 100755
--- a/.jenkins.d/30-coverage.sh
+++ b/.jenkins.d/30-coverage.sh
@@ -1,10 +1,5 @@
#!/usr/bin/env bash
-set -e
-
-JDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
-source "$JDIR"/util.sh
-
-set -x
+set -ex
if [[ $JOB_NAME == *"code-coverage" ]]; then
gcovr --object-directory=build \
diff --git a/.jenkins.d/README.md b/.jenkins.d/README.md
index 5813349..e8dbf37 100644
--- a/.jenkins.d/README.md
+++ b/.jenkins.d/README.md
@@ -1,36 +1,28 @@
-CONTINUOUS INTEGRATION SCRIPTS
-==============================
+# CONTINUOUS INTEGRATION SCRIPTS
-Environment Variables Used in Build Scripts
--------------------------------------------
+## Environment Variables Used in Build Scripts
-- `NODE_LABELS`: the variable defines a list of OS properties. The set values are used by the
- build scripts to select proper behavior for different OS.
+- `NODE_LABELS`: space-separated list of platform properties. The included values are used by
+ the build scripts to select the proper behavior for different operating systems and versions.
- The list should include at least `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`.
+ The list should normally contain `[OS_TYPE]`, `[DISTRO_TYPE]`, and `[DISTRO_VERSION]`.
- Possible values for Linux:
+ Example values:
- * `[OS_TYPE]`: `Linux`
- * `[DISTRO_TYPE]`: `Ubuntu`
- * `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`
+ - `[OS_TYPE]`: `Linux`, `OSX`
+ - `[DISTRO_TYPE]`: `Ubuntu`, `CentOS`
+ - `[DISTRO_VERSION]`: `Ubuntu-16.04`, `Ubuntu-18.04`, `CentOS-8`, `OSX-10.14`, `OSX-10.15`
- Possible values for OS X / macOS:
-
- * `[OS_TYPE]`: `OSX`
- * `[DISTRO_TYPE]`: `OSX` (can be absent)
- * `[DISTRO_VERSION]`: `OSX-10.11`, `OSX-10.12`, `OSX-10.13`
-
-- `JOB_NAME`: optional variable to define type of the job. Depending on the defined job type,
+- `JOB_NAME`: optional variable that defines the type of build job. Depending on the job type,
the build scripts can perform different tasks.
Possible values:
- * empty: default build process
- * `code-coverage` (Ubuntu Linux is assumed): debug build with tests and code coverage analysis
- * `limited-build`: only a single debug build with tests
+ - empty: default build task
+ - `code-coverage`: debug build with tests and code coverage analysis (Ubuntu Linux is assumed)
+ - `limited-build`: only a single debug build with tests
-- `CACHE_DIR`: the variable defines a path to folder containing cached files from previous builds,
- e.g., a compiled version of ndn-cxx library. If not set, `/tmp` is used.
+- `CACHE_DIR`: directory containing cached files from previous builds, e.g., a compiled version
+ of ndn-cxx. If not set, `/tmp` is used.
-- `WAF_JOBS`: number of parallel build jobs used by waf, defaults to 1.
+- `WAF_JOBS`: number of parallel build threads used by waf, defaults to 1.
diff --git a/.jenkins.d/util.sh b/.jenkins.d/util.sh
index 8ddc4ba..8077a74 100644
--- a/.jenkins.d/util.sh
+++ b/.jenkins.d/util.sh
@@ -16,6 +16,7 @@
set ${saved_xtrace}
return ${ret}
}
+export -f has
sudo_preserve_env() {
local saved_xtrace
@@ -35,3 +36,4 @@
set ${saved_xtrace}
sudo env "${vars[@]}" "$@"
}
+export -f sudo_preserve_env