Add compilation tests and static analysis on the Gitlab shared runner.

.gitlab-ci.yml

@@ -106,3 +106,8 @@ test-mergerq-job-dnf:
     - make -j
     - make -j -C tests check-programs
     - sudo -E make check
+
+include:
+  - local: .gitlab/ci/gitlab-shared-docker.yml
+  - local: .gitlab/ci/compilation-gcc.gitlab-ci.yml
+  - local: .gitlab/ci/compilation-clang.gitlab-ci.yml

.gitlab/ci/cibuild-setup-ubuntu.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+set -ex
+
+PACKAGES=(
+	git make autoconf automake autopoint pkg-config libtool libtool-bin
+	gettext libssl-dev libdevmapper-dev libpopt-dev uuid-dev libsepol1-dev
+	libjson-c-dev libssh-dev libblkid-dev tar libargon2-0-dev libpwquality-dev
+	sharutils dmsetup jq xxd expect keyutils netcat passwd openssh-client sshpass
+)
+
+COMPILER="${COMPILER:?}"
+COMPILER_VERSION="${COMPILER_VERSION:?}"
+
+grep -E '^deb' /etc/apt/sources.list > /etc/apt/sources.list~
+sed -Ei 's/^deb /deb-src /' /etc/apt/sources.list~
+cat /etc/apt/sources.list~ >> /etc/apt/sources.list
+
+apt-get -y update --fix-missing
+DEBIAN_FRONTEND=noninteractive apt-get -yq install software-properties-common wget lsb-release
+if [[ $COMPILER == "gcc" ]]; then
+  # Latest gcc stack deb packages provided by
+  # https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test
+  add-apt-repository -y ppa:ubuntu-toolchain-r/test
+elif [[ $COMPILER == "clang" ]]; then
+  .gitlab/ci/llvm.sh $COMPILER_VERSION
+  # scan-build
+  PACKAGES+=(clang-tools-$COMPILER_VERSION)
+  PACKAGES+=(perl)
+else
+  exit 1
+fi
+
+PACKAGES+=(${COMPILER}-$COMPILER_VERSION)
+
+DEBIAN_FRONTEND=noninteractive apt-get -yq install "${PACKAGES[@]}"
+apt-get -y build-dep cryptsetup
+
+echo "====================== VERSIONS ==================="
+if [[ $COMPILER == "clang" ]]; then
+  scan-build${COMPILER_VERSION:+-$COMPILER_VERSION} --help
+fi
+
+${COMPILER}-$COMPILER_VERSION -v
+echo "====================== END VERSIONS ==================="

.gitlab/ci/clang-Wall

@@ -0,0 +1,70 @@
+#!/bin/bash
+# clang -Wall plus other important warnings not included in -Wall
+
+for arg in "$@"
+do
+    case $arg in
+	-O*) Wuninitialized=-Wuninitialized;;  # only makes sense with `-O'
+    esac
+done
+
+CLANG="clang${COMPILER_VERSION:+-$COMPILER_VERSION}"
+
+#PEDANTIC="-std=gnu99"
+#PEDANTIC="-pedantic -std=gnu99"
+#PEDANTIC="-pedantic -std=gnu99 -Wno-variadic-macros"
+#CONVERSION="-Wconversion"
+
+EXTRA="-Wextra \
+ -Wsign-compare \
+ -Werror-implicit-function-declaration \
+ -Wpointer-arith \
+ -Wwrite-strings \
+ -Wswitch \
+ -Wmissing-format-attribute \
+ -Winit-self \
+ -Wdeclaration-after-statement \
+ -Wold-style-definition \
+ -Wno-missing-field-initializers \
+ -Wno-unused-parameter \
+ -Wno-attributes \
+ -Wno-long-long"
+
+exec $CLANG $PEDANTIC $CONVERSION \
+	-Wall $Wuninitialized \
+        -Wno-switch \
+	-Wdisabled-optimization \
+	-Wwrite-strings \
+	-Wpointer-arith \
+	-Wbad-function-cast \
+	-Wmissing-prototypes \
+	-Wmissing-declarations \
+	-Wstrict-prototypes \
+	-Wnested-externs \
+	-Wcomment \
+	-Winline \
+	-Wcast-align \
+	-Wcast-qual \
+	-Wredundant-decls $EXTRA \
+	"$@" 2>&1 | {
+	if [[ $USE_FILTER -eq 1 ]]; then
+		.gitlab/ci/warnings_filter.py
+	else
+		cat
+	fi
+}
+
+
+#	-fcheck-memory-usage \
+#    -Wall implies:
+#	-Wshadow
+#	-Wimplicit
+#	-Wreturn-type
+#	-Wunused
+#	-Wswitch
+#	-Wformat
+#	-Wchar-subscripts
+#	-Wparentheses
+#	-Wmissing-braces
+#	-Wtraditional
+#	-Wid-clash-31

.gitlab/ci/clang-setup-ubuntu.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -ex
+
+PACKAGES=(
+	git make autoconf automake autopoint pkg-config libtool libtool-bin
+	gettext libssl-dev libdevmapper-dev libpopt-dev uuid-dev libsepol1-dev
+	libjson-c-dev libssh-dev libblkid-dev tar libargon2-0-dev libpwquality-dev
+	sharutils dmsetup jq xxd expect keyutils netcat passwd openssh-client sshpass
+)
+
+COMPILER="${COMPILER:?}"
+COMPILER_VERSION="${COMPILER_VERSION:?}"
+
+grep -E '^deb' /etc/apt/sources.list > /etc/apt/sources.list~
+sed -Ei 's/^deb /deb-src /' /etc/apt/sources.list~
+cat /etc/apt/sources.list~ >> /etc/apt/sources.list
+#bash -c "echo 'deb-src http://archive.ubuntu.com/ubuntu/ $RELEASE main restricted universe multiverse' >>/etc/apt/sources.list"
+
+apt-get -y update --fix-missing
+DEBIAN_FRONTEND=noninteractive apt-get -yq install software-properties-common
+
+# Latest gcc stack deb packages provided by
+# https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test
+add-apt-repository -y ppa:ubuntu-toolchain-r/test
+PACKAGES+=(gcc-$COMPILER_VERSION)
+
+DEBIAN_FRONTEND=noninteractive apt-get -yq install "${PACKAGES[@]}"
+apt-get -y build-dep cryptsetup

.gitlab/ci/compilation-clang.gitlab-ci.yml

@@ -0,0 +1,25 @@
+test-clang-compilation:
+  extends:
+    - .gitlab-shared-clang
+  script:
+    - export CFLAGS="-Wall -Werror"
+    - ./configure --enable-pwquality --enable-libargon2
+    - make -j
+
+# Clang doesn't support json output, so we cannot use the warnings filter
+# test-clang-Wall-script:
+#   extends:
+#     - .gitlab-shared-clang
+#   script:
+#     - export CFLAGS="-g -O0"
+#     - export CC=".gitlab/ci/clang-Wall"
+#     - ./configure --enable-pwquality --enable-libargon2
+#     - make -j CFLAGS="-g -O0 -Werror"
+
+test-scan-build:
+  extends:
+    - .gitlab-shared-clang
+  script:
+    - scan-build${COMPILER_VERSION:+-$COMPILER_VERSION} -V ./configure CFLAGS="-g -O0" --enable-internal-sse-argon2 --enable-pwquality --enable-libargon2
+    - make clean
+    - scan-build${COMPILER_VERSION:+-$COMPILER_VERSION} -maxloop 10 make -j

.gitlab/ci/compilation-gcc.gitlab-ci.yml

@@ -0,0 +1,24 @@
+test-gcc-compilation:
+  extends:
+    - .gitlab-shared-gcc
+  script:
+    - export CFLAGS="-Wall -Werror"
+    - ./configure --enable-pwquality --enable-libargon2
+    - make -j
+
+test-gcc-Wall-script:
+  extends:
+    - .gitlab-shared-gcc
+  script:
+    - export CFLAGS="-g -O0"
+    - export CC=".gitlab/ci/gcc-Wall"
+    - USE_FILTER=0 ./configure --enable-pwquality --enable-libargon2
+    - USE_FILTER=1 make -j CFLAGS="-g -O0 -fdiagnostics-format=json"
+
+test-gcc-fanalyzer:
+  extends:
+    - .gitlab-shared-gcc
+  script:
+    - export CFLAGS="-Wall -Werror -g -O0 -fanalyzer -fdiagnostics-path-format=separate-events"
+    - ./configure --enable-pwquality --enable-libargon2
+    - make -j

.gitlab/ci/gcc-Wall

@@ -0,0 +1,75 @@
+#!/bin/bash
+# gcc -Wall plus other important warnings not included in -Wall
+
+for arg in "$@"
+do
+    case $arg in
+	-O*) Wuninitialized=-Wuninitialized;;  # only makes sense with `-O'
+    esac
+done
+
+GCC="gcc${COMPILER_VERSION:+-$COMPILER_VERSION}"
+
+#PEDANTIC="-std=gnu99"
+#PEDANTIC="-pedantic -std=gnu99"
+#PEDANTIC="-pedantic -std=gnu99 -Wno-variadic-macros"
+#CONVERSION="-Wconversion"
+# -Wpacked \
+
+
+EXTRA="-Wextra \
+ -Wsign-compare \
+ -Werror-implicit-function-declaration \
+ -Wpointer-arith \
+ -Wwrite-strings \
+ -Wswitch \
+ -Wmissing-format-attribute \
+ -Wstrict-aliasing=3 \
+ -Winit-self \
+ -Wunsafe-loop-optimizations \
+ -Wdeclaration-after-statement \
+ -Wold-style-definition \
+ -Wno-missing-field-initializers \
+ -Wno-unused-parameter \
+ -Wno-attributes \
+ -Wno-long-long
+ -Wmaybe-uninitialized
+ -Wvla"
+
+exec $GCC $PEDANTIC $CONVERSION \
+	-Wall $Wuninitialized \
+        -Wno-switch \
+	-Wdisabled-optimization \
+	-Wwrite-strings \
+	-Wpointer-arith \
+	-Wbad-function-cast \
+	-Wmissing-prototypes \
+	-Wmissing-declarations \
+	-Wstrict-prototypes \
+	-Wnested-externs \
+	-Wcomment \
+	-Winline \
+	-Wcast-align \
+	-Wcast-qual \
+	-Wredundant-decls $EXTRA \
+	"$@" 2>&1 | {
+	if [[ $USE_FILTER -eq 1 ]]; then
+		.gitlab/ci/warnings_filter.py
+	else
+		cat
+	fi
+}
+
+#	-fcheck-memory-usage \
+#    -Wall implies:
+#	-Wshadow
+#	-Wimplicit
+#	-Wreturn-type
+#	-Wunused
+#	-Wswitch
+#	-Wformat
+#	-Wchar-subscripts
+#	-Wparentheses
+#	-Wmissing-braces
+#	-Wtraditional
+#	-Wid-clash-31

.gitlab/ci/gitlab-shared-docker.yml

@@ -0,0 +1,29 @@
+.gitlab-shared-docker:
+  image: ubuntu:focal
+  tags:
+    - gitlab-org-docker
+  stage: test
+  interruptible: true
+  rules:
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  before_script:
+    - .gitlab/ci/cibuild-setup-ubuntu.sh
+    - export CC="${COMPILER}${COMPILER_VERSION:+-$COMPILER_VERSION}"
+    - export CXX="${COMPILER}++${COMPILER_VERSION:+-$COMPILER_VERSION}"
+    - ./autogen.sh
+
+.gitlab-shared-gcc:
+  extends:
+    - .gitlab-shared-docker
+  variables:
+    COMPILER: "gcc"
+    COMPILER_VERSION: "11"
+    RUN_SSH_PLUGIN_TEST: "1"
+
+.gitlab-shared-clang:
+  extends:
+    - .gitlab-shared-docker
+  variables:
+    COMPILER: "clang"
+    COMPILER_VERSION: "13"
+    RUN_SSH_PLUGIN_TEST: "1"

.gitlab/ci/llvm.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+################################################################################
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+################################################################################
+#
+# This script will install the llvm toolchain on the different
+# Debian and Ubuntu versions
+#
+# from: https://apt.llvm.org/llvm.sh
+
+set -eux
+
+# Check for required tools
+needed_binaries=(lsb_release wget add-apt-repository)
+missing_binaries=()
+for binary in "${needed_binaries[@]}"; do
+    if ! which $binary &>/dev/null ; then
+        missing_binaries+=($binary)
+    fi
+done
+if [[ ${#missing_binaries[@]} -gt 0 ]] ; then
+    echo "You are missing some tools this script requires: ${missing_binaries[@]}"
+    echo "(hint: apt install lsb-release wget software-properties-common)"
+    exit 4
+fi
+
+# read optional command line argument
+LLVM_VERSION=13
+if [ "$#" -eq 1 ]; then
+    LLVM_VERSION=$1
+fi
+
+DISTRO=$(lsb_release -is)
+VERSION=$(lsb_release -sr)
+DIST_VERSION="${DISTRO}_${VERSION}"
+
+if [[ $EUID -ne 0 ]]; then
+   echo "This script must be run as root!"
+   exit 1
+fi
+
+declare -A LLVM_VERSION_PATTERNS
+LLVM_VERSION_PATTERNS[9]="-9"
+LLVM_VERSION_PATTERNS[10]="-10"
+LLVM_VERSION_PATTERNS[11]="-11"
+LLVM_VERSION_PATTERNS[12]="-12"
+LLVM_VERSION_PATTERNS[13]="-13"
+LLVM_VERSION_PATTERNS[14]=""
+
+if [ ! ${LLVM_VERSION_PATTERNS[$LLVM_VERSION]+_} ]; then
+    echo "This script does not support LLVM version $LLVM_VERSION"
+    exit 3
+fi
+
+LLVM_VERSION_STRING=${LLVM_VERSION_PATTERNS[$LLVM_VERSION]}
+
+# find the right repository name for the distro and version
+case "$DIST_VERSION" in
+    Debian_9* )       REPO_NAME="deb http://apt.llvm.org/stretch/  llvm-toolchain-stretch$LLVM_VERSION_STRING main" ;;
+    Debian_10* )      REPO_NAME="deb http://apt.llvm.org/buster/   llvm-toolchain-buster$LLVM_VERSION_STRING  main" ;;
+    Debian_11* )      REPO_NAME="deb http://apt.llvm.org/bullseye/ llvm-toolchain-bullseye$LLVM_VERSION_STRING  main" ;;
+    Debian_unstable ) REPO_NAME="deb http://apt.llvm.org/unstable/ llvm-toolchain$LLVM_VERSION_STRING         main" ;;
+    Debian_testing )  REPO_NAME="deb http://apt.llvm.org/unstable/ llvm-toolchain$LLVM_VERSION_STRING         main" ;;
+    Ubuntu_16.04 )    REPO_NAME="deb http://apt.llvm.org/xenial/   llvm-toolchain-xenial$LLVM_VERSION_STRING  main" ;;
+    Ubuntu_18.04 )    REPO_NAME="deb http://apt.llvm.org/bionic/   llvm-toolchain-bionic$LLVM_VERSION_STRING  main" ;;
+    Ubuntu_18.10 )    REPO_NAME="deb http://apt.llvm.org/cosmic/   llvm-toolchain-cosmic$LLVM_VERSION_STRING  main" ;;
+    Ubuntu_19.04 )    REPO_NAME="deb http://apt.llvm.org/disco/    llvm-toolchain-disco$LLVM_VERSION_STRING   main" ;;
+    Ubuntu_19.10 )   REPO_NAME="deb http://apt.llvm.org/eoan/      llvm-toolchain-eoan$LLVM_VERSION_STRING    main" ;;
+    Ubuntu_20.04 )   REPO_NAME="deb http://apt.llvm.org/focal/     llvm-toolchain-focal$LLVM_VERSION_STRING   main" ;;
+    Ubuntu_21.04 )   REPO_NAME="deb http://apt.llvm.org/groovy/    llvm-toolchain-groovy$LLVM_VERSION_STRING  main" ;;
+    Ubuntu_21.10 )   REPO_NAME="deb http://apt.llvm.org/hirsute/   llvm-toolchain-hirsute$LLVM_VERSION_STRING main" ;;
+    * )
+        echo "Distribution '$DISTRO' in version '$VERSION' is not supported by this script (${DIST_VERSION})."
+        exit 2
+esac
+
+
+# install everything
+wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
+add-apt-repository "${REPO_NAME}"
+apt-get update
+apt-get install -y clang-$LLVM_VERSION lldb-$LLVM_VERSION lld-$LLVM_VERSION clangd-$LLVM_VERSION

.gitlab/ci/warnings_filter.py

@@ -0,0 +1,31 @@
+#!/usr/bin/python3
+
+import sys
+import json
+import linecache
+
+if __name__ == "__main__":
+    json_string = sys.stdin.read()
+    if json_string in [None, ""]:
+        sys.exit(0)
+
+    parsed = json.loads(json_string)
+    #print(json.dumps(parsed, indent=4, sort_keys=True))
+
+    r = 0
+
+    for o in parsed:
+        kind = o["kind"]
+
+        start = o["locations"][0]["caret"]
+        l = linecache.getline(start["file"], int(start["line"]))
+
+        ignored = "json_object_object_foreach" in l
+
+        print(f"{o['kind']} {'ignored' if ignored else 'FOUND'} in {start['file']}:{start['line']}:{start['column']} {o['message']}")
+        print(f"line contains:\n\t{l}", end="")
+
+        if not ignored:
+            r = 1
+
+    sys.exit(r)