Rebase the Python install and patch binaries

Author: niess

.gitignore

@@ -2,4 +2,5 @@ __pycache__
 *.pyc
 *.pyo
 appimage/*.AppImage
+share/excludelist
 wiki

.travis/Dockerfile

@@ -8,7 +8,7 @@ RUN apt update -y -qq                                                       && \
     apt install --no-install-recommends -y -qq                                 \
         build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev \
         libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev     \
-        libexpat1-dev liblzma-dev libffi-dev tk-dev libfuse2 wget           && \
+        libexpat1-dev liblzma-dev libffi-dev tk-dev automake libfuse2 wget  && \
     rm -rf /var/lib/apt/lists/*
 
 COPY . /work/.travis

.travis/script.sh

@@ -12,9 +12,9 @@ if [ -n "${OPENSSL}" ]; then
 fi
 
 
-DEBUG=true ./appimage/build-plugin.sh
-DEBUG=true ./appimage/build-python.sh python2
-DEBUG=true ./appimage/build-python.sh python3
-DEBUG=true ./appimage/build-python.sh scipy
+./appimage/build-plugin.sh
+./appimage/build-python.sh python2
+./appimage/build-python.sh python3
+./appimage/build-python.sh scipy
 
 python3 -m tests

appimage/build-plugin.sh

@@ -9,6 +9,7 @@ fi
 
 EXEC_NAME="${EXEC_NAME:-linuxdeploy-plugin-python}"
 ARCH="${ARCH:-$(arch)}"
+PATCHELF_VERSION="0.10"
 
 REPO_ROOT=$(readlink -f $(dirname "$0")"/..")
 
@@ -30,6 +31,32 @@ fi
 pushd $BUILD_DIR
 
 
+# Install the ELF patcher
+prefix="${PWD}/AppDir/usr"
+mkdir -p "${prefix}"
+wget -cq --no-check-certificate "https://github.com/NixOS/patchelf/archive/${PATCHELF_VERSION}.tar.gz"
+tar -xzf "${PATCHELF_VERSION}.tar.gz"
+pushd "patchelf-${PATCHELF_VERSION}"
+./bootstrap.sh
+if [ "$ARCH" == "i386" ]; then
+    ./configure --prefix="${prefix}" --build=i686-pc-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32
+elif [ "$ARCH" == "x86_64" ]; then
+    ./configure --prefix="${prefix}"
+fi
+make -j$(nproc)
+make install
+popd
+rm -rf "AppDir/usr/share"
+strip "AppDir/usr/bin/patchelf"
+
+
+# Package the exclusion list
+mkdir -p "AppDir/share"
+pushd "AppDir/share"
+wget -cq --no-check-certificate "https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist"
+popd
+
+
 # Build the AppImage
 appimagetool="appimagetool-${ARCH}.AppImage"
 
@@ -39,7 +66,6 @@ if [ ! -f "${appimagetool}" ]; then
     chmod u+x "${appimagetool}"
 fi
 
-mkdir -p "AppDir/share"
 cp "${REPO_ROOT}/${EXEC_NAME}.sh" "AppDir/AppRun"
 chmod +x "AppDir/AppRun"
 cp -r "${REPO_ROOT}/share" "AppDir"

appimage/build-python.sh

@@ -70,11 +70,9 @@ sed -i "s|[{][{]exe[}][}]|${exe}|g" "${name}.desktop"
 sed -i "s|[{][{]name[}][}]|${name}|g" "${name}.desktop"
 
 ./"${linuxdeploy}" --appdir AppDir \
-                   --plugin python
-./"${linuxdeploy}" --appdir AppDir \
+                   --plugin python \
                    -i "${REPO_ROOT}/appimage/resources/python.png" \
                    -d "${name}.desktop" \
-                   -e "AppDir/usr/bin/.${exe}" \
                    --custom-apprun "AppDir/usr/bin/${exe}" \
                    --output "appimage"
 

linuxdeploy-plugin-python.sh

@@ -119,61 +119,120 @@ if [[ "${source_dir}" == *.tgz ]] || [[ "${source_dir}" == *.tar.gz ]]; then
     source_dir="$filename"
 fi
 
+prefix="usr/python"
 cd "${source_dir}"
-./configure ${PYTHON_CONFIG} "--with-ensurepip=install" "--prefix=/usr"
+./configure ${PYTHON_CONFIG} "--with-ensurepip=install" "--prefix=/${prefix}"
 HOME="${PYTHON_BUILD_DIR}" make -j"$NPROC" DESTDIR="$APPDIR" install
 
 
-# Copy any TCl/Tk shared data
-if [ -d "/usr/share/tcltk" ]; then
-    cp -r "/usr/share/tcltk" "${APPDIR}/usr/share/tcltk"
-fi
-
-
 # Install any extra requirements with pip
 if [ ! -z "${PIP_REQUIREMENTS}" ]; then
-    cd "${APPDIR}/usr"
+    cd "${APPDIR}/${prefix}/bin"
     pythons=( "python"?"."? )
-    HOME="${PYTHON_BUILD_DIR}" PYTHONHOME=$PWD ./bin/${pythons[0]} -m pip install ${PIP_OPTIONS} ${PIP_REQUIREMENTS}
+    HOME="${PYTHON_BUILD_DIR}" PYTHONHOME=$(readlink -f ${PWD}/..) ./${pythons[0]} -m pip install ${PIP_OPTIONS} ${PIP_REQUIREMENTS}
 fi
 
 
 # Prune the install
-cd "$APPDIR/usr"
+cd "$APPDIR/${prefix}"
 rm -rf "bin/python"*"-config" "bin/idle"* "include" "lib/pkgconfig" \
        "share/doc" "share/man" "lib/libpython"*".a" "lib/python"*"/test" \
        "lib/python"*"/config-"*"-x86_64-linux-gnu"
 
 
 # Wrap the Python executables
-cd "$APPDIR/usr/bin"
+cd "$APPDIR/${prefix}/bin"
 set +e
 pythons=$(ls "python" "python"? "python"?"."? "python"?"."?"m" 2>/dev/null)
 set -e
+cd "$APPDIR/usr/bin"
 for python in $pythons
 do
     if [ ! -L "$python" ]; then
-        mv "$python" ".$python"
+        strip "$APPDIR/${prefix}/bin/${python}"
         cp "${BASEDIR}/share/python-wrapper.sh" "$python"
         sed -i "s|[{][{]PYTHON[}][}]|$python|g" "$python"
+        sed -i "s|[{][{]PREFIX[}][}]|$prefix|g" "$python"
     fi
 done
 
 
 # Sanitize the shebangs of local Python scripts
-for exe in $(ls "${APPDIR}/usr/bin"*)
+cd "$APPDIR/${prefix}/bin"
+for exe in $(ls "${APPDIR}/${prefix}/bin"*)
 do
-    sed -i '1s|^#!.*\(python[0-9.]*\)|#!/bin/sh\n"exec" "$(dirname $(readlink -f $\{0\}))/\1" "$0" "$@"|' "$exe"
+    sed -i '1s|^#!.*\(python[0-9.]*\)|#!/bin/sh\n"exec" "$(dirname $(readlink -f $\{0\}))/../../bin/\1" "$0" "$@"|' "$exe"
 done
 
 
 # Set a hook in Python for cleaning the path detection
-cp "$BASEDIR/share/sitecustomize.py" "$APPDIR"/usr/lib/python*/site-packages
+cp "$BASEDIR/share/sitecustomize.py" "$APPDIR"/${prefix}/lib/python*/site-packages
+
 
+# Patch binaries and install dependencies
+excludelist=$(cat "${BASEDIR}/share/excludelist" | sed 's|#.*||g' | sed -r '/^\s*$/d')
 
-# Relocate the Python extension modules for the dynamic linker
+is_excluded () {
+    local e
+    for e in ${excludelist}; do
+        [[ "$e" == "$1" ]] && echo "true" && return 0
+    done
+    return 0
+}
+
+set +e
+patchelf=$(command -v patchelf)
+set -e
+patchelf="${patchelf:-${BASEDIR}/usr/bin/patchelf}"
+
+patch_binary() {
+    local name="$(basename $1)"
+
+    if [ "${name::3}" == "lib" ]; then
+        if [ ! -f "${APPDIR}/usr/lib/${name}" ]; then
+            echo "Patching dependency ${name}"
+            strip "$1"
+            "${patchelf}" --set-rpath '$ORIGIN' "$1"
+            ln -rs "$1" "${APPDIR}/usr/lib"
+        fi
+    else
+        echo "Patching C-extension module ${name}"
+        strip "$1"
+
+        local rel=$(dirname $(readlink -f $1))
+        rel=${rel#${APPDIR}/usr}
+        rel=$(echo $rel | sed 's|/[_a-zA-Z0-9.-]*|/..|g')
+        "${patchelf}" --set-rpath '$ORIGIN:$ORIGIN'"${rel}/lib" "$1"
+    fi
+
+    local deps
+    for deps in $(ldd $1); do
+        if [[ "${deps::1}" == "/" ]] && [[ "${deps}" != "${APPDIR}"* ]]; then
+            local lib="$(basename ${deps})"
+            if [ ! -f "${APPDIR}/usr/lib/${lib}" ]; then
+                if [ ! "$(is_excluded ${lib})" ]; then
+                    echo "Installing dependency ${lib}"
+                    cp "${deps}" "${APPDIR}/usr/lib"
+                    strip "${APPDIR}/usr/lib/${lib}"
+                    "${patchelf}" --set-rpath '$ORIGIN' "${APPDIR}/usr/lib/${lib}"
+                fi
+            fi
+        fi
+    done
+    return 0
+}
+
+cd "$APPDIR/${prefix}/bin"
 python=$(ls "python"?"."?)
-cd "$APPDIR/usr/lib"
-mv "${python}/lib-dynload/"* "."
-rm -rf "${python}/lib-dynload"
-ln -fs "../" "${python}/lib-dynload"
+mkdir -p "${APPDIR}/usr/lib"
+cd "${APPDIR}/${prefix}/lib/${python}"
+find "lib-dynload" -name '*.so' -type f | while read file; do patch_binary "${file}"; done
+find "site-packages" -name '*.so' -type f | while read file; do patch_binary "${file}"; done
+find "site-packages" -name 'lib*.so*' -type f | while read file; do patch_binary "${file}"; done
+
+
+# Copy any TCl/Tk shared data
+if [[ -d "/usr/share/tcltk" ]] && [[ ! -d "${APPDIR}/${prefix}/share/tcltk" ]]; then
+    mkdir -p "${APPDIR}/${prefix}/share"
+    cp -r "/usr/share/tcltk" "${APPDIR}/${prefix}/share"
+fi

share/python-wrapper.sh

@@ -1,13 +1,14 @@
 #!/bin/bash
 
 # Configure the environment
-export TCL_LIBRARY="${APPDIR}/usr/share/tcltk/tcl8.6"
-export TK_LIBRARY="${APPDIR}/usr/share/tcltk/tk8.6"
+prefix="{{PREFIX}}"
+export TCL_LIBRARY="${APPDIR}/${prefix}/share/tcltk/tcl"*
+export TK_LIBRARY="${APPDIR}/${prefix}/share/tcltk/tk"*
 export TKPATH="${TK_LIBRARY}"
 
 # Resolve symlinks within the image
 nickname="{{PYTHON}}"
-executable="${APPDIR}/usr/bin/${nickname}"
+executable="${APPDIR}/${prefix}/bin/${nickname}"
 if [ -L "${executable}" ]; then
     nickname="$(basename $(readlink -f ${executable}))"
 fi
@@ -18,7 +19,7 @@ do
     if [[ "${opt}" =~ "I" ]] || [[ "${opt}" =~ "E" ]]; then
         # Environment variables are disabled ($PYTHONHOME). Let's run in a safe
         # mode from the raw Python binary inside the AppImage
-        "$APPDIR/usr/bin/.${nickname}" "$@"
+        "$APPDIR/${prefix}/bin/${nickname}" "$@"
         exit "$?"
     fi
 done
@@ -33,5 +34,5 @@ fi
 # Wrap the call to Python in order to mimic a call from the source
 # executable ($ARGV0), but potentially located outside of the Python
 # install ($PYTHONHOME)
-(PYTHONHOME="${APPDIR}/usr" exec -a "${executable}" "$APPDIR/usr/bin/.${nickname}" "$@")
+(PYTHONHOME="${APPDIR}/${prefix}" exec -a "${executable}" "$APPDIR/${prefix}/bin/${nickname}" "$@")
 exit "$?"

tests/test_plugin.py

@@ -115,11 +115,12 @@ def check_base(self, tag):
         v = [int(vi) for vi in version.split(".")]
         self.assertEqual(cfg["version"][:3], v)
         self.assertEqual(cfg["executable"], python)
-        self.assertEqual(cfg["prefix"], os.path.join(cfg["appdir"], "usr"))
+        self.assertEqual(cfg["prefix"], os.path.join(cfg["appdir"], "usr",
+            "python"))
         site_packages = os.path.join("lib",
             "python{:}.{:}".format(*cfg["version"][:2]), "site-packages")
         self.assertEqual(cfg["path"][-1], os.path.join(cfg["appdir"],
-            "usr", site_packages))
+            "usr", "python", site_packages))
         user_packages = os.path.join(cfg["home"], ".local", site_packages)
         self.assertTrue(user_packages in cfg["path"])