Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add clightning support #21

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ services:
env:
global:
- DOCKERBUILDER=greenaddress/core_builder_for_android
- DOCKERHASH=06e3e394f41bd389a11ce51a2bb4ec46ff082e86d25f4f90644aad5b4faaf0f1
- DOCKERHASH=9e8e2dd9ea172a500e106a87fb116976e466e14d5cbac4307759961b946977a1
- REPO=https://github.com/bitcoin/bitcoin.git
- COMMIT=1bc9988993ee84bc814e5a7f33cc90f670a19f6a
- TARGETHOST=aarch64-linux-android
Expand Down
2 changes: 1 addition & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
FROM debian:buster@sha256:41f76363fd83982e14f7644486e1fb04812b3894aa4e396137c3435eaf05de88
FROM debian:buster@sha256:79f0b1682af1a6a29ff63182c8103027f4de98b22d8fb50040e9c4bb13e3de78
COPY /buster_deps.sh /
RUN /buster_deps.sh
2 changes: 1 addition & 1 deletion buster_deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ sha256_file=8381c440fe61fcbb01e209211ac01b519cd6adf51ab1c2281d5daad6ca4c8c8c

apt-get -yqq update &> /dev/null
apt-get -yqq upgrade &> /dev/null
apt-get -yqq install python curl build-essential libtool autotools-dev automake pkg-config bsdmainutils unzip git &> /dev/null
apt-get -yqq install python python3-{pip,virtualenv,mako} curl build-essential libtool autotools-dev automake pkg-config bsdmainutils unzip git gettext &> /dev/null

mkdir -p /opt

Expand Down
99 changes: 80 additions & 19 deletions fetchbuild.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,30 @@ configextra=$5
target_host=$6
bits=$7


unpackdep() {
curl -sL -o tmp.tar.gz $1
echo "$2 tmp.tar.gz" | sha256sum --check
tar xzf tmp.tar.gz
rm tmp.tar.gz
archive=$(basename $1)
curl -sL -o ${archive} $1
echo "$2 ${archive}" | sha256sum --check
tar xf ${archive}
rm ${archive}
}

# build lightning deps
LNBUILDROOT=$PWD/ln_build_root
mkdir $LNBUILDROOT

export ANDROID_NDK_HOME=/opt/android-ndk-r20b
export PATH=$ANDROID_NDK_HOME/toolchains/llvm/prebuilt/linux-x86_64/bin:${PATH}
export AR=${target_host/v7a/}-ar
export AS=${target_host}21-clang
export CC=${target_host}21-clang
export CXX=${target_host}21-clang++
export LD=${target_host/v7a/}-ld
export AR=${target_host/v7a}-ar
export AS=${target_host}24-clang
export CC=${target_host}24-clang
export CXX=${target_host}24-clang++
export LD=${target_host/v7a}-ld
export STRIP=${target_host/v7a}-strip
export CFLAGS="-flto"
export LDFLAGS="$CFLAGS -pie -static-libstdc++ -fuse-ld=lld"
export LDFLAGS="-pie"
export MAKE_HOST=${target_host/v7a}
export HOST=${target_host/v7a}
export CONFIGURATOR_CC="/usr/bin/gcc"

NDKARCH=arm
if [ "$target_host" = "i686-linux-android" ]; then
Expand All @@ -38,16 +43,67 @@ elif [ "$target_host" = "aarch64-linux-android" ]; then
NDKARCH=arm64
fi

NDKV=19
if [ "$bits" = "64" ]; then
NDKV=21
fi

num_jobs=4
if [ -f /proc/cpuinfo ]; then
num_jobs=$(grep ^processor /proc/cpuinfo | wc -l)
fi

## build cln first

# sqlite
unpackdep https://www.sqlite.org/2018/sqlite-autoconf-3260000.tar.gz 5daa6a3fb7d1e8c767cd59c4ded8da6e4b00c61d3b466d0685e35c4dd6d7bf5d
cd sqlite-autoconf-3260000
./configure --enable-static --disable-readline --disable-threadsafe --host=${target_host} CC=$CC --prefix=${LNBUILDROOT}
make -j $num_jobs
make install
cd ..
rm -rf sqlite-autoconf-3260000

# gmp
unpackdep https://gmplib.org/download/gmp/gmp-6.1.2.tar.bz2 5275bb04f4863a13516b2f39392ac5e272f5e1bb8057b18aec1c9b79d73d8fb2
cd gmp-6.1.2
./configure --enable-static --disable-assembly --host=${target_host} CC=$CC --prefix=${LNBUILDROOT}
make -j $num_jobs
make install
cd ..
rm -rf gmp-6.1.2

# download lightning
git clone https://github.com/ElementsProject/lightning.git lightning
cd lightning
git checkout v0.7.3

# set virtualenv for lightning
python3 -m virtualenv venv
. venv/bin/activate
pip install -r requirements.txt

# set standard cc for the configurator
sed -i 's/$CC ${CWARNFLAGS-$BASE_WARNFLAGS} $CDEBUGFLAGS $COPTFLAGS -o $CONFIGURATOR $CONFIGURATOR.c/$CONFIGURATOR_CC ${CWARNFLAGS-$BASE_WARNFLAGS} $CDEBUGFLAGS $COPTFLAGS -o $CONFIGURATOR $CONFIGURATOR.c/g' configure
sed -i 's/-Wno-maybe-uninitialized/-Wno-uninitialized/g' configure
./configure CONFIGURATOR_CC=${CONFIGURATOR_CC} --prefix=${LNBUILDROOT} --disable-developer --disable-compat --disable-valgrind --enable-static
Comment on lines +82 to +84
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This would be a nice patch to upstream. The configurator is always a bit weird in cross-compile scenarios fwiw


cp /repo/lightning-gen_header_versions.h gen_header_versions.h
# update arch based on toolchain
sed "s'NDKCOMPILER'${CC}'" /repo/lightning-config.vars > config.vars
sed "s'NDKCOMPILER'${CC}'" /repo/lightning-config.h > ccan/config.h
Comment on lines +88 to +89
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Strange switch of sed separators. Any particular reason for not continuing to use / like above?

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

the code used to contain slashes in sed, that was fixed but the separator wasn't changed

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ack, sed syntax could be aligned in the entire script. I don't really like using sed on pre-generated headers file, but it is a dirty trick to don't use qemu in the CI.


# patch makefile
patch -p1 < /repo/lightning-makefile.patch
patch -p1 < /repo/lightning-jsonrpc.patch
patch -p1 < /repo/lightning-endian.patch

# build external libraries and source
make PIE=1 DEVELOPER=0 || echo "continue"
make clean -C ccan/ccan/cdump/tools
make LDFLAGS="" CC="${CONFIGURATOR_CC}" LDLIBS="-L/usr/local/lib" -C ccan/ccan/cdump/tools
make PIE=1 DEVELOPER=0
deactivate
cd ..


export CFLAGS="-flto"
export LDFLAGS="$CFLAGS -pie -static-libstdc++ -fuse-ld=lld"
# build core
git clone $repo ${reponame}
cd ${reponame}
Expand Down Expand Up @@ -129,7 +185,12 @@ cd ..
if [ "${reponame}" != "${rename}" ]; then
mv ${reponame}/depends/${target_host/v7a/}/bin/${reponame}d ${reponame}/depends/${target_host/v7a/}/bin/${rename}d
mv ${reponame}/depends/${target_host/v7a/}/bin/${reponame}-cli ${reponame}/depends/${target_host/v7a/}/bin/${rename}-cli
tar -Jcf /repo/${target_host/v7a/}_${rename}.tar.xz -C ${reponame}/depends/${target_host/v7a/}/bin ${rename}d ${rename}-cli tor
outputtar=/repo/${target_host/v7a/}_${rename}.tar
else
tar -Jcf /repo/${target_host/v7a/}_$(basename $(dirname ${repo})).tar.xz -C ${reponame}/depends/${target_host/v7a/}/bin ${rename}d ${rename}-cli tor
outputtar=/repo/${target_host/v7a/}_$(basename $(dirname ${repo})).tar
fi
tar -cf ${outputtar} -C ${reponame}/depends/${target_host/v7a/}/bin ${rename}d ${rename}-cli tor
tar -rf ${outputtar} -C lightning/lightningd lightning_channeld lightning_closingd lightning_connectd lightning_gossipd lightning_hsmd lightning_onchaind lightning_openingd lightningd
tar -rf ${outputtar} -C lightning plugins/autoclean plugins/fundchannel plugins/pay
tar -rf ${outputtar} -C lightning/cli lightning-cli
xz ${outputtar}
90 changes: 90 additions & 0 deletions lightning-config.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/* Generated by CCAN configurator */
#ifndef CCAN_CONFIG_H
#define CCAN_CONFIG_H
#ifndef _GNU_SOURCE
#define _GNU_SOURCE /* Always use GNU extensions. */
#endif
#define CCAN_COMPILER "NDKCOMPILER"
#define CCAN_CFLAGS "-Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition -Werror -std=gnu11 -g -fstack-protector -Og"
#define CCAN_OUTPUT_EXE_CFLAG "-o"

#define HAVE_CCAN 1
#define HAVE_32BIT_OFF_T 0
#define HAVE_ALIGNOF 1
#define HAVE_ASPRINTF 1
#define HAVE_ATTRIBUTE_COLD 1
#define HAVE_ATTRIBUTE_CONST 1
#define HAVE_ATTRIBUTE_DEPRECATED 0
#define HAVE_ATTRIBUTE_NONNULL 1
#define HAVE_ATTRIBUTE_SENTINEL 1
#define HAVE_ATTRIBUTE_PURE 1
#define HAVE_ATTRIBUTE_MAY_ALIAS 1
#define HAVE_ATTRIBUTE_NORETURN 1
#define HAVE_ATTRIBUTE_PRINTF 1
#define HAVE_ATTRIBUTE_UNUSED 1
#define HAVE_ATTRIBUTE_USED 1
#define HAVE_BACKTRACE 0
#define HAVE_BIG_ENDIAN 0
#define HAVE_BSWAP_64 0
#define HAVE_BUILTIN_CHOOSE_EXPR 1
#define HAVE_BUILTIN_CLZ 1
#define HAVE_BUILTIN_CLZL 1
#define HAVE_BUILTIN_CLZLL 1
#define HAVE_BUILTIN_CTZ 1
#define HAVE_BUILTIN_CTZL 1
#define HAVE_BUILTIN_CTZLL 1
#define HAVE_BUILTIN_CONSTANT_P 1
#define HAVE_BUILTIN_EXPECT 1
#define HAVE_BUILTIN_FFS 1
#define HAVE_BUILTIN_FFSL 1
#define HAVE_BUILTIN_FFSLL 1
#define HAVE_BUILTIN_POPCOUNT 1
#define HAVE_BUILTIN_POPCOUNTL 1
#define HAVE_BUILTIN_POPCOUNTLL 1
#define HAVE_BUILTIN_TYPES_COMPATIBLE_P 1
#define HAVE_ICCARM_INTRINSICS 0
#define HAVE_BYTESWAP_H 0
#define HAVE_CLOCK_GETTIME 1
#define HAVE_CLOCK_GETTIME_IN_LIBRT 0
#define HAVE_COMPOUND_LITERALS 1
#define HAVE_FCHDIR 1
#define HAVE_ERR_H 1
#define HAVE_FILE_OFFSET_BITS 0
#define HAVE_FOR_LOOP_DECLARATION 1
#define HAVE_FLEXIBLE_ARRAY_MEMBER 1
#define HAVE_GETPAGESIZE 1
#define HAVE_ISBLANK 1
#define HAVE_LITTLE_ENDIAN 1
#define HAVE_MEMMEM 1
#define HAVE_MEMRCHR 0
#define HAVE_MMAP 1
#define HAVE_PROC_SELF_MAPS 0
#define HAVE_QSORT_R_PRIVATE_LAST 0
#define HAVE_STRUCT_TIMESPEC 1
#define HAVE_SECTION_START_STOP 1
#define HAVE_STACK_GROWS_UPWARDS 0
#define HAVE_STATEMENT_EXPR 1
#define HAVE_SYS_FILIO_H 1
#define HAVE_SYS_TERMIOS_H 0
#define HAVE_SYS_UNISTD_H 1
#define HAVE_TYPEOF 1
#define HAVE_UNALIGNED_ACCESS 1
#define HAVE_UTIME 1
#define HAVE_WARN_UNUSED_RESULT 1
#define HAVE_OPENMP 0
#define HAVE_VALGRIND_MEMCHECK_H 0
#define HAVE_UCONTEXT 0
#define HAVE_POINTER_SAFE_MAKECONTEXT 0
#define HAVE_BUILTIN_CPU_SUPPORTS 1
#define HAVE_GOOD_LIBSODIUM 0
#define HAVE_SQLITE3_EXPANDED_SQL 1
#define HAVE_SQLITE3 1
#define HAVE_POSTGRES 0
#define HAVE_GCC 0
#define HAVE_MODERN_GCC 0
#define HAVE_PWRITEV 0
#endif /* CCAN_CONFIG_H */
#define DEVELOPER 0
#define EXPERIMENTAL_FEATURES 0
#define COMPAT 0
#include "../ccan_compat.h"
88 changes: 88 additions & 0 deletions lightning-config.vars
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
HAVE_32BIT_OFF_T=0
HAVE_ALIGNOF=1
HAVE_ASPRINTF=1
HAVE_ATTRIBUTE_COLD=1
HAVE_ATTRIBUTE_CONST=1
HAVE_ATTRIBUTE_DEPRECATED=0
HAVE_ATTRIBUTE_NONNULL=1
HAVE_ATTRIBUTE_SENTINEL=1
HAVE_ATTRIBUTE_PURE=1
HAVE_ATTRIBUTE_MAY_ALIAS=1
HAVE_ATTRIBUTE_NORETURN=1
HAVE_ATTRIBUTE_PRINTF=1
HAVE_ATTRIBUTE_UNUSED=1
HAVE_ATTRIBUTE_USED=1
HAVE_BACKTRACE=0
HAVE_BIG_ENDIAN=0
HAVE_BSWAP_64=0
HAVE_BUILTIN_CHOOSE_EXPR=1
HAVE_BUILTIN_CLZ=1
HAVE_BUILTIN_CLZL=1
HAVE_BUILTIN_CLZLL=1
HAVE_BUILTIN_CTZ=1
HAVE_BUILTIN_CTZL=1
HAVE_BUILTIN_CTZLL=1
HAVE_BUILTIN_CONSTANT_P=1
HAVE_BUILTIN_EXPECT=1
HAVE_BUILTIN_FFS=1
HAVE_BUILTIN_FFSL=1
HAVE_BUILTIN_FFSLL=1
HAVE_BUILTIN_POPCOUNT=1
HAVE_BUILTIN_POPCOUNTL=1
HAVE_BUILTIN_POPCOUNTLL=1
HAVE_BUILTIN_TYPES_COMPATIBLE_P=1
HAVE_ICCARM_INTRINSICS=0
HAVE_BYTESWAP_H=0
HAVE_CLOCK_GETTIME=1
HAVE_CLOCK_GETTIME_IN_LIBRT=0
HAVE_COMPOUND_LITERALS=1
HAVE_FCHDIR=1
HAVE_ERR_H=1
HAVE_FILE_OFFSET_BITS=0
HAVE_FOR_LOOP_DECLARATION=1
HAVE_FLEXIBLE_ARRAY_MEMBER=1
HAVE_GETPAGESIZE=1
HAVE_ISBLANK=1
HAVE_LITTLE_ENDIAN=1
HAVE_MEMMEM=1
HAVE_MEMRCHR=0
HAVE_MMAP=1
HAVE_PROC_SELF_MAPS=0
HAVE_QSORT_R_PRIVATE_LAST=0
HAVE_STRUCT_TIMESPEC=1
HAVE_SECTION_START_STOP=1
HAVE_STACK_GROWS_UPWARDS=0
HAVE_STATEMENT_EXPR=1
HAVE_SYS_FILIO_H=1
HAVE_SYS_TERMIOS_H=0
HAVE_SYS_UNISTD_H=1
HAVE_TYPEOF=1
HAVE_UNALIGNED_ACCESS=1
HAVE_UTIME=1
HAVE_WARN_UNUSED_RESULT=1
HAVE_OPENMP=0
HAVE_VALGRIND_MEMCHECK_H=0
HAVE_UCONTEXT=0
HAVE_POINTER_SAFE_MAKECONTEXT=0
HAVE_BUILTIN_CPU_SUPPORTS=1
HAVE_GOOD_LIBSODIUM=0
HAVE_SQLITE3_EXPANDED_SQL=1
HAVE_SQLITE3=1
HAVE_POSTGRES=0
HAVE_GCC=0
HAVE_MODERN_GCC=0
HAVE_PWRITEV=0
PREFIX=/ln_build_root
CC=NDKCOMPILER
CONFIGURATOR_CC=/usr/bin/gcc
CWARNFLAGS=-Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition -Werror
CDEBUGFLAGS=-std=gnu11 -g -fstack-protector
COPTFLAGS=-Og
VALGRIND=0
DEVELOPER=0
EXPERIMENTAL_FEATURES=0
COMPAT=0
PYTEST=
STATIC=1
ASAN=0
TEST_NETWORK=regtest
17 changes: 17 additions & 0 deletions lightning-endian.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
diff --git a/ccan/ccan/endian/endian.h b/ccan/ccan/endian/endian.h
index 3753f490..6e9d8046 100644
--- a/ccan/ccan/endian/endian.h
+++ b/ccan/ccan/endian/endian.h
@@ -104,8 +104,12 @@ static inline uint64_t bswap_64(uint64_t val)
#endif

/* Needed for Glibc like endiness check */
+#ifndef __LITTLE_ENDIAN
#define __LITTLE_ENDIAN 1234
+#endif
+#ifndef __BIG_ENDIAN
#define __BIG_ENDIAN 4321
+#endif
Comment on lines +8 to +14
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is this change required for? It might need upstreaming as well, since we should be able to detect endiannes reliably.

Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point we should try to give it a go without

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Configurator has an option about HAVE_BIG_ENDIAN, but I am not sure solve the issue during building ccan https://github.com/ElementsProject/lightning/blob/359433f374472a6128993bbaace2c1c7c62ee107/ccan/tools/configurator/configurator.c#L174

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The issue imho is that the configurator is running on the host architecture and will thus use the host endianness instead of the target arch's endianness. Maybe ElementsProject/lightning#3464 could be used to run the configurator in qemu emulating the target arch, thus picking up the correct values automatically?


/* Sanity check the defines. We don't handle weird endianness. */
#if !HAVE_LITTLE_ENDIAN && !HAVE_BIG_ENDIAN
29 changes: 29 additions & 0 deletions lightning-gen_header_versions.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/* Generated file by tools/headerversions, do not edit! */
/* GMP version: 6.1.2 */
/* SQLITE3 version: 3024000 */
/* ZLIB version: 1.2.11 */
#include <ccan/err/err.h>
#include <gmp.h>
#include <sqlite3.h>
#include <zlib.h>

static inline void check_linked_library_versions(void)
{
char compiled_gmp_version[100];
if (SQLITE_VERSION_NUMBER != sqlite3_libversion_number())
errx(1, "SQLITE version mismatch: compiled %u, now %u",
SQLITE_VERSION_NUMBER, sqlite3_libversion_number());
/* zlib documents that first char alters ABI. Kudos! */
if (zlibVersion()[0] != ZLIB_VERSION[0])
errx(1, "zlib version mismatch: compiled %s, now %s",
ZLIB_VERSION, zlibVersion());
/* GMP doesn't say anything, and we have to assemble our own string. */
snprintf(compiled_gmp_version, sizeof(compiled_gmp_version),
"%u.%u.%u",
__GNU_MP_VERSION,
__GNU_MP_VERSION_MINOR,
__GNU_MP_VERSION_PATCHLEVEL);
if (strcmp(compiled_gmp_version, gmp_version) != 0)
errx(1, "gmp version mismatch: compiled %s, now %s",
compiled_gmp_version, gmp_version);
}
Loading