New Scraper and NN

.travis.yml

@@ -1,5 +1,5 @@
 sudo: required
-dist: trusty
+dist: bionic
 os: linux
 language: minimal
 cache:
@@ -22,25 +22,30 @@ env:
     - WINEDEBUG=fixme-all
   matrix:
 # ARM
-    - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" GRIDCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports RUN_TESTS=false"
+    - HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf zlib1g-dev" DEP_OPTS="NO_QT=1" CHECK_DOC=1 GOAL="install" GRIDCOIN_CONFIG="--enable-glibc-back-compat --disable-tests RUN_TESTS=false"
 # Win32
     - HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="python3 nsis g++-mingw-w64-i686 wine1.6" RUN_TESTS=false GOAL="" GRIDCOIN_CONFIG="--enable-reduce-exports"
 # Qt5 & system libs
-    - HOST=x86_64-unknown-linux-gnu PACKAGES="libqt5gui5 libqt5core5a qtbase5-dev libqt5dbus5 qttools5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libminiupnpc-dev protobuf-compiler libqrencode-dev xvfb" NO_DEPENDS=1 NEED_XVFB=1 RUN_TESTS=true GOAL="install" GRIDCOIN_CONFIG=" --with-incompatible-bdb --enable-reduce-exports --with-gui=qt5" 
+    - HOST=x86_64-unknown-linux-gnu PACKAGES="libqt5gui5 libqt5core5a qtbase5-dev libqt5dbus5 qttools5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-iostreams-dev libboost-program-options-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev protobuf-compiler libqrencode-dev xvfb zlib1g-dev" NO_DEPENDS=1 NEED_XVFB=1 RUN_TESTS=true GOAL="install" GRIDCOIN_CONFIG=" --with-incompatible-bdb --enable-reduce-exports --with-gui=qt5" 
 # 32-bit + dash
     - HOST=i686-pc-linux-gnu PACKAGES="g++-multilib" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="install" GRIDCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash"
 # Win64
-    - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine1.6 xvfb" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="" GRIDCOIN_CONFIG="--enable-reduce-exports" NEED_XVFB=1
+    - HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" PACKAGES="gcc-7 python3 nsis g++-mingw-w64-x86-64 wine1.6 xvfb" DEP_OPTS="NO_QT=1" RUN_TESTS=false GOAL="" GRIDCOIN_CONFIG="--enable-reduce-exports" NEED_XVFB=1
 # Cross-Mac - disabled for now, since we cannot compile QtSVG with the SDK
     #- HOST=x86_64-apple-darwin11 PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python-dev" GRIDCOIN_CONFIG="--enable-gui --enable-reduce-exports" OSX_SDK=10.11 RUN_TESTS=false GOAL="deploy"
 
 before_install:
     - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
+    - sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
+    - sudo apt-get update -q
+    - sudo apt-get install gcc-7 -y
 install:
     - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi
     - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi
     - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi
 before_script:
+    - if [ "$HOST" == "i686-w64-mingw32" ]; then sudo update-alternatives --set i686-w64-mingw32-g++ /usr/bin/i686-w64-mingw32-g++-posix; fi
+    - if [ "$HOST" == "x86_64-w64-mingw32" ]; then sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix; fi
     - unset CC; unset CXX
     - mkdir -p depends/SDKs depends/sdk-sources
     - if [ -n "$OSX_SDK" -a ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz; fi

README.md

@@ -13,6 +13,7 @@ These dependencies are required:
 
  Library     | Purpose          | Description
  ------------|------------------|----------------------------------------------------------------
+ pkg-config  | Build            | Learn library inter-dependencies
  libssl      | Crypto           | Random Number Generation, Elliptic Curve Cryptography
  libboost    | Utility          | Library for threading, data structures, etc
  libevent    | Networking       | OS independent asynchronous networking

build-aux/m4/ax_boost_iostreams.m4

@@ -0,0 +1,116 @@
+# ===========================================================================
+#    https://www.gnu.org/software/autoconf-archive/ax_boost_iostreams.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_BOOST_IOSTREAMS
+#
+# DESCRIPTION
+#
+#   Test for IOStreams library from the Boost C++ libraries. The macro
+#   requires a preceding call to AX_BOOST_BASE. Further documentation is
+#   available at <http://randspringer.de/boost/index.html>.
+#
+#   This macro calls:
+#
+#     AC_SUBST(BOOST_IOSTREAMS_LIB)
+#
+#   And sets:
+#
+#     HAVE_BOOST_IOSTREAMS
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Thomas Porschberg <[email protected]>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 21
+
+AC_DEFUN([AX_BOOST_IOSTREAMS],
+[
+	AC_ARG_WITH([boost-iostreams],
+	AS_HELP_STRING([--with-boost-iostreams@<:@=special-lib@:>@],
+                   [use the IOStreams library from boost - it is possible to specify a certain library for the linker
+                        e.g. --with-boost-iostreams=boost_iostreams-gcc-mt-d-1_33_1 ]),
+        [
+        if test "$withval" = "no"; then
+			want_boost="no"
+        elif test "$withval" = "yes"; then
+            want_boost="yes"
+            ax_boost_user_iostreams_lib=""
+        else
+		    want_boost="yes"
+		ax_boost_user_iostreams_lib="$withval"
+		fi
+        ],
+        [want_boost="yes"]
+	)
+
+	if test "x$want_boost" = "xyes"; then
+        AC_REQUIRE([AC_PROG_CC])
+		CPPFLAGS_SAVED="$CPPFLAGS"
+		CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+		export CPPFLAGS
+
+		LDFLAGS_SAVED="$LDFLAGS"
+		LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+		export LDFLAGS
+
+        AC_CACHE_CHECK(whether the Boost::IOStreams library is available,
+					   ax_cv_boost_iostreams,
+        [AC_LANG_PUSH([C++])
+		 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/iostreams/filtering_stream.hpp>
+											 @%:@include <boost/range/iterator_range.hpp>
+											]],
+                                  [[std::string  input = "Hello World!";
+								 namespace io = boost::iostreams;
+									 io::filtering_istream  in(boost::make_iterator_range(input));
+									 return 0;
+                                   ]])],
+                             ax_cv_boost_iostreams=yes, ax_cv_boost_iostreams=no)
+         AC_LANG_POP([C++])
+		])
+		if test "x$ax_cv_boost_iostreams" = "xyes"; then
+			AC_DEFINE(HAVE_BOOST_IOSTREAMS,,[define if the Boost::IOStreams library is available])
+            BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+            if test "x$ax_boost_user_iostreams_lib" = "x"; then
+                for libextension in `ls $BOOSTLIBDIR/libboost_iostreams*.so* $BOOSTLIBDIR/libboost_iostream*.dylib* $BOOSTLIBDIR/libboost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_iostreams.*\)\.so.*$;\1;' -e 's;^lib\(boost_iostream.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_iostreams.*\)\.a.*$;\1;'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
+                                 [link_iostreams="no"])
+				done
+                if test "x$link_iostreams" != "xyes"; then
+                for libextension in `ls $BOOSTLIBDIR/boost_iostreams*.dll* $BOOSTLIBDIR/boost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_iostreams.*\)\.dll.*$;\1;' -e 's;^\(boost_iostreams.*\)\.a.*$;\1;'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
+                                 [link_iostreams="no"])
+				done
+                fi
+
+            else
+               for ax_lib in $ax_boost_user_iostreams_lib boost_iostreams-$ax_boost_user_iostreams_lib; do
+				      AC_CHECK_LIB($ax_lib, main,
+                                   [BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
+                                   [link_iostreams="no"])
+                  done
+
+            fi
+            if test "x$ax_lib" = "x"; then
+                AC_MSG_ERROR(Could not find a version of the library!)
+            fi
+			if test "x$link_iostreams" != "xyes"; then
+				AC_MSG_ERROR(Could not link against $ax_lib !)
+			fi
+		fi
+
+		CPPFLAGS="$CPPFLAGS_SAVED"
+	LDFLAGS="$LDFLAGS_SAVED"
+	fi
+])

build-aux/m4/ax_boost_zlib.m4

@@ -0,0 +1,90 @@
+# SYNOPSIS
+#
+#   AX_BOOST_ZLIB
+#
+# DESCRIPTION
+#
+#   Test for ZLib library from the Boost C++ libraries. The macro
+#   requires a preceding call to AX_BOOST_BASE. Further documentation is
+#   available at <http://randspringer.de/boost/index.html>.
+#
+#   This macro calls:
+#
+#     AC_SUBST(BOOST_ZLIB_LIB)
+#
+#   And sets:
+#
+#     HAVE_BOOST_ZLIB
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Thomas Porschberg <[email protected]>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 21
+
+AC_DEFUN([AX_BOOST_ZLIB],
+[
+	AC_ARG_WITH([boost-zlib],
+	AS_HELP_STRING([--with-boost-zlib@<:@=special-lib@:>@],
+                   [use the ZLib library from boost - it is possible to specify a certain library for the linker
+                        e.g. --with-boost-zlib=boost_zlib-gcc-mt-d-1_33_1 ]),
+        [
+        if test "$withval" = "no"; then
+			want_boost="no"
+        elif test "$withval" = "yes"; then
+            want_boost="yes"
+            ax_boost_user_zlib_lib=""
+        else
+		    want_boost="yes"
+		ax_boost_user_zlib_lib="$withval"
+		fi
+        ],
+        [want_boost="yes"]
+	)
+
+	if test "x$want_boost" = "xyes"; then
+        AC_REQUIRE([AC_PROG_CC])
+		CPPFLAGS_SAVED="$CPPFLAGS"
+		CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+		export CPPFLAGS
+
+		LDFLAGS_SAVED="$LDFLAGS"
+		LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+		export LDFLAGS
+
+			AC_DEFINE(HAVE_BOOST_ZLIB,,[define if the Boost::zlib library is available])
+            BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+            if test "x$ax_boost_user_zlib_lib" = "x"; then
+                for libextension in `ls $BOOSTLIBDIR/libboost_zlib*.so* $BOOSTLIBDIR/libboost_iostream*.dylib* $BOOSTLIBDIR/libboost_zlib*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_zlib.*\)\.so.*$;\1;' -e 's;^lib\(boost_iostream.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_zlib.*\)\.a.*$;\1;'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_ZLIB_LIB="-l$ax_lib"; AC_SUBST(BOOST_ZLIB_LIB) link_zlib="yes"; break],
+                                 [link_zlib="no"])
+				done
+                if test "x$link_zlib" != "xyes"; then
+                for libextension in `ls $BOOSTLIBDIR/boost_zlib*.dll* $BOOSTLIBDIR/boost_zlib*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_zlib.*\)\.dll.*$;\1;' -e 's;^\(boost_zlib.*\)\.a.*$;\1;'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_ZLIB_LIB="-l$ax_lib"; AC_SUBST(BOOST_ZLIB_LIB) link_zlib="yes"; break],
+                                 [link_zlib="no"])
+				done
+                fi
+
+            else
+               for ax_lib in $ax_boost_user_zlib_lib boost_zlib-$ax_boost_user_zlib_lib; do
+				      AC_CHECK_LIB($ax_lib, main,
+                                   [BOOST_ZLIB_LIB="-l$ax_lib"; AC_SUBST(BOOST_ZLIB_LIB) link_zlib="yes"; break],
+                                   [link_zlib="no"])
+                  done
+
+            fi
+
+		CPPFLAGS="$CPPFLAGS_SAVED"
+	LDFLAGS="$LDFLAGS_SAVED"
+	fi
+])

build-aux/m4/m4_ax_boost_iostreams.m4

@@ -0,0 +1,116 @@
+# ===========================================================================
+#    https://www.gnu.org/software/autoconf-archive/ax_boost_iostreams.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_BOOST_IOSTREAMS
+#
+# DESCRIPTION
+#
+#   Test for IOStreams library from the Boost C++ libraries. The macro
+#   requires a preceding call to AX_BOOST_BASE. Further documentation is
+#   available at <http://randspringer.de/boost/index.html>.
+#
+#   This macro calls:
+#
+#     AC_SUBST(BOOST_IOSTREAMS_LIB)
+#
+#   And sets:
+#
+#     HAVE_BOOST_IOSTREAMS
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Thomas Porschberg <[email protected]>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved. This file is offered as-is, without any
+#   warranty.
+
+#serial 21
+
+AC_DEFUN([AX_BOOST_IOSTREAMS],
+[
+	AC_ARG_WITH([boost-iostreams],
+	AS_HELP_STRING([--with-boost-iostreams@<:@=special-lib@:>@],
+                   [use the IOStreams library from boost - it is possible to specify a certain library for the linker
+                        e.g. --with-boost-iostreams=boost_iostreams-gcc-mt-d-1_33_1 ]),
+        [
+        if test "$withval" = "no"; then
+			want_boost="no"
+        elif test "$withval" = "yes"; then
+            want_boost="yes"
+            ax_boost_user_iostreams_lib=""
+        else
+		    want_boost="yes"
+		ax_boost_user_iostreams_lib="$withval"
+		fi
+        ],
+        [want_boost="yes"]
+	)
+
+	if test "x$want_boost" = "xyes"; then
+        AC_REQUIRE([AC_PROG_CC])
+		CPPFLAGS_SAVED="$CPPFLAGS"
+		CPPFLAGS="$CPPFLAGS $BOOST_CPPFLAGS"
+		export CPPFLAGS
+
+		LDFLAGS_SAVED="$LDFLAGS"
+		LDFLAGS="$LDFLAGS $BOOST_LDFLAGS"
+		export LDFLAGS
+
+        AC_CACHE_CHECK(whether the Boost::IOStreams library is available,
+					   ax_cv_boost_iostreams,
+        [AC_LANG_PUSH([C++])
+		 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[@%:@include <boost/iostreams/filtering_stream.hpp>
+											 @%:@include <boost/range/iterator_range.hpp>
+											]],
+                                  [[std::string  input = "Hello World!";
+								 namespace io = boost::iostreams;
+									 io::filtering_istream  in(boost::make_iterator_range(input));
+									 return 0;
+                                   ]])],
+                             ax_cv_boost_iostreams=yes, ax_cv_boost_iostreams=no)
+         AC_LANG_POP([C++])
+		])
+		if test "x$ax_cv_boost_iostreams" = "xyes"; then
+			AC_DEFINE(HAVE_BOOST_IOSTREAMS,,[define if the Boost::IOStreams library is available])
+            BOOSTLIBDIR=`echo $BOOST_LDFLAGS | sed -e 's/@<:@^\/@:>@*//'`
+            if test "x$ax_boost_user_iostreams_lib" = "x"; then
+                for libextension in `ls $BOOSTLIBDIR/libboost_iostreams*.so* $BOOSTLIBDIR/libboost_iostream*.dylib* $BOOSTLIBDIR/libboost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^lib\(boost_iostreams.*\)\.so.*$;\1;' -e 's;^lib\(boost_iostream.*\)\.dylib.*$;\1;' -e 's;^lib\(boost_iostreams.*\)\.a.*$;\1;'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
+                                 [link_iostreams="no"])
+				done
+                if test "x$link_iostreams" != "xyes"; then
+                for libextension in `ls $BOOSTLIBDIR/boost_iostreams*.dll* $BOOSTLIBDIR/boost_iostreams*.a* 2>/dev/null | sed 's,.*/,,' | sed -e 's;^\(boost_iostreams.*\)\.dll.*$;\1;' -e 's;^\(boost_iostreams.*\)\.a.*$;\1;'` ; do
+                     ax_lib=${libextension}
+				    AC_CHECK_LIB($ax_lib, exit,
+                                 [BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
+                                 [link_iostreams="no"])
+				done
+                fi
+
+            else
+               for ax_lib in $ax_boost_user_iostreams_lib boost_iostreams-$ax_boost_user_iostreams_lib; do
+				      AC_CHECK_LIB($ax_lib, main,
+                                   [BOOST_IOSTREAMS_LIB="-l$ax_lib"; AC_SUBST(BOOST_IOSTREAMS_LIB) link_iostreams="yes"; break],
+                                   [link_iostreams="no"])
+                  done
+
+            fi
+            if test "x$ax_lib" = "x"; then
+                AC_MSG_ERROR(Could not find a version of the library!)
+            fi
+			if test "x$link_iostreams" != "xyes"; then
+				AC_MSG_ERROR(Could not link against $ax_lib !)
+			fi
+		fi
+
+		CPPFLAGS="$CPPFLAGS_SAVED"
+	LDFLAGS="$LDFLAGS_SAVED"
+	fi
+])